{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"02eaf587-125d-a4fe-62c7-3dac63b6c75f"},"source":"Following a similar recipe to lewis' R script (https://www.kaggle.com/cartographic/bosch-production-line-performance/bish-bash-xgboost), sampling the data to select features before running on the full set in order to stay within kaggle's memory limits. Here I add in the train_date data too.\n\nPlease feel free to fork and improve."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b8ff0786-470f-f3de-e826-2334abc61cda"},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import matthews_corrcoef, roc_auc_score\nfrom sklearn.cross_validation import cross_val_score, StratifiedKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.linear_model import LogisticRegression\n%matplotlib inline"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"eb108407-22f9-9716-1905-d6ebfd6a5b59"},"outputs":[],"source":"# I'm limited by RAM here and taking the first N rows is likely to be\n# a bad idea for the date data since it is ordered.\n# Sample the data in a roundabout way:\ndate_chunks = pd.read_csv(\"../input/train_date.csv\", index_col=0, chunksize=100000, dtype=np.float32)\nnum_chunks = pd.read_csv(\"../input/train_numeric.csv\", index_col=0,\n                         usecols=list(range(969)), chunksize=100000, dtype=np.float32)\nX = pd.concat([pd.concat([dchunk, nchunk], axis=1).sample(frac=0.05)\n               for dchunk, nchunk in zip(date_chunks, num_chunks)])\ny = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, usecols=[0,969], dtype=np.float32).loc[X.index].values.ravel()\nX = X.values"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7fa0923e-ddfa-d046-f7a8-842667883110"},"outputs":[],"source":"clf = XGBClassifier(base_score=0.005)\nclf.fit(X, y)\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c6159cc6-47b5-e632-b0b6-2f8e0578876d"},"outputs":[],"source":"# threshold for a manageable number of features\nplt.hist(clf.feature_importances_[clf.feature_importances_>0])\nimportant_indices = np.where(clf.feature_importances_>0.005)[0]\nprint(important_indices)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d2a8fa06-d180-d316-2f92-aef473339726"},"outputs":[],"source":"# load entire dataset for these features. \n# note where the feature indices are split so we can load the correct ones straight from read_csv\nn_date_features = 1156\nX = np.concatenate([\n    pd.read_csv(\"../input/train_date.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices < n_date_features] + 1])).values,\n    pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices >= n_date_features] + 1 - 1156])).values\n], axis=1)\ny = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32, usecols=[0,969]).values.ravel()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0dd2d440-bb20-82c5-7519-d1eb24a7e510"},"outputs":[],"source":"clf = XGBClassifier(max_depth=5, base_score=0.005)\ncv = StratifiedKFold(y, n_folds=3)\npreds = np.ones(y.shape[0])\nfor i, (train, test) in enumerate(cv):\n    preds[test] = clf.fit(X[train], y[train]).predict_proba(X[test])[:,1]\n    print(\"fold {}, ROC AUC: {:.3f}\".format(i, roc_auc_score(y[test], preds[test])))\nprint(roc_auc_score(y, preds))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"eca4bd3d-586a-233d-6dcd-4380939e2b1e"},"outputs":[],"source":"# pick the best threshold out-of-fold\nthresholds = np.linspace(0.01, 0.99, 50)\nmcc = np.array([matthews_corrcoef(y, preds>thr) for thr in thresholds])\nplt.plot(thresholds, mcc)\nbest_threshold = thresholds[mcc.argmax()]\nprint(mcc.max())"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0e9aceeb-3b06-74af-3238-c500a33d5a71"},"outputs":[],"source":"# load test data\nX = np.concatenate([\n    pd.read_csv(\"../input/test_date.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices<1156]+1])).values,\n    pd.read_csv(\"../input/test_numeric.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices>=1156] +1 - 1156])).values\n], axis=1)\n\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a796c10a-d149-0608-350b-6adb5c733e8b"},"outputs":[],"source":"\n# generate predictions at the chosen threshold\npreds = (clf.predict_proba(X)[:,1] > best_threshold).astype(np.int8)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e0574c4a-ece0-6545-fbc9-cafe21d0844c"},"outputs":[],"source":"\n\nclf = LogisticRegression()\ncv = StratifiedKFold(y,n_folds=3)\npreds = np.ones(y.shape[0])\n\nfor i, (train, test) in enumerate(cv):\n    preds[test] = clf.fit(X_not_nan[train], y[train]).predict_proba(X_not_nan[test])[:,1]\n    print(\"fold {}, ROC AUC: {:.3f}\".format(i, roc_auc_score(y[test], preds[test])))\nprint(roc_auc_score(y, preds))\n    "},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b7985a1d-2709-c6c5-80ea-0e2cc8f1731b"},"outputs":[],"source":"# pick the best threshold out-of-fold\nthresholds = np.linspace(0.01, 0.99, 50)\nmcc = np.array([matthews_corrcoef(y, preds>thr) for thr in thresholds])\nplt.plot(thresholds, mcc)\nbest_threshold = thresholds[mcc.argmax()]\nprint(mcc.max())"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c50c738e-dcd4-bc30-2017-e2b2d0008d97"},"outputs":[],"source":"# and submit\nsub = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\nsub[\"Response\"] = preds\nsub.to_csv(\"submission.csv.gz\", compression=\"gzip\")"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9ee1414e-1de6-3976-1d92-fc27e36651a0"},"outputs":[],"source":"\nX.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f4500238-f15a-37ce-ee47-8c2296e88291"},"outputs":[],"source":"important_indices"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4e4291e1-ce9c-12f4-b696-fe600de715a6"},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7826534b-d31e-925f-8bc7-4ab64f384f79"},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"eb5fb541-1bbf-f18f-1f1f-9631ef2e80e4"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}