{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"02eaf587-125d-a4fe-62c7-3dac63b6c75f"},"source":"Following a similar recipe to lewis' R script (https://www.kaggle.com/cartographic/bosch-production-line-performance/bish-bash-xgboost), sampling the data to select features before running on the full set in order to stay within kaggle's memory limits. Here I add in the train_date data too.\n\nPlease feel free to fork and improve."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b8ff0786-470f-f3de-e826-2334abc61cda"},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import matthews_corrcoef, roc_auc_score\nfrom sklearn.cross_validation import cross_val_score, StratifiedKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ea38b99e-6c93-a77b-f2c0-94848a7bcd7e"},"outputs":[],"source":"date_chunks"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"eb108407-22f9-9716-1905-d6ebfd6a5b59"},"outputs":[],"source":"# I'm limited by RAM here and taking the first N rows is likely to be\n# a bad idea for the date data since it is ordered.\n# Sample the data in a roundabout way:\ndate_chunks = pd.read_csv(\"../input/train_date.csv\", index_col=0, chunksize=100000, dtype=np.float32)\nnum_chunks = pd.read_csv(\"../input/train_numeric.csv\", index_col=0,\n                         usecols=list(range(969)), chunksize=100000, dtype=np.float32)\nX = pd.concat([pd.concat([dchunk, nchunk], axis=1).sample(frac=0.05)\n               for dchunk, nchunk in zip(date_chunks, num_chunks)])\ny = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, usecols=[0,969], dtype=np.float32).loc[X.index].values.ravel()\nX = X.values"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7fa0923e-ddfa-d046-f7a8-842667883110"},"outputs":[],"source":"clf = XGBClassifier(base_score=0.005)\nclf.fit(X, y)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c6159cc6-47b5-e632-b0b6-2f8e0578876d"},"outputs":[],"source":"# threshold for a manageable number of features\nplt.hist(clf.feature_importances_[clf.feature_importances_>0])\nimportant_indices = np.where(clf.feature_importances_>0.005)[0]\nprint(important_indices)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d2a8fa06-d180-d316-2f92-aef473339726"},"outputs":[],"source":"# load entire dataset for these features. \n# note where the feature indices are split so we can load the correct ones straight from read_csv\nn_date_features = 1156\nX = np.concatenate([\n    pd.read_csv(\"../input/train_date.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices < n_date_features] + 1])).values,\n    pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices >= n_date_features] + 1 - 1156])).values\n], axis=1)\ny = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32, usecols=[0,969]).values.ravel()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0dd2d440-bb20-82c5-7519-d1eb24a7e510"},"outputs":[],"source":"clf = XGBClassifier(max_depth=5, base_score=0.005)\ncv = StratifiedKFold(y, n_folds=3)\npreds = np.ones(y.shape[0])\nfor i, (train, test) in enumerate(cv):\n    preds[test] = clf.fit(X[train], y[train]).predict_proba(X[test])[:,1]\n    print(\"fold {}, ROC AUC: {:.3f}\".format(i, roc_auc_score(y[test], preds[test])))\nprint(roc_auc_score(y, preds))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"eca4bd3d-586a-233d-6dcd-4380939e2b1e"},"outputs":[],"source":"# pick the best threshold out-of-fold\nthresholds = np.linspace(0.01, 0.99, 50)\nmcc = np.array([matthews_corrcoef(y, preds>thr) for thr in thresholds])\nplt.plot(thresholds, mcc)\nbest_threshold = thresholds[mcc.argmax()]\nprint(mcc.max())"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0e9aceeb-3b06-74af-3238-c500a33d5a71"},"outputs":[],"source":"# load test data\nX = np.concatenate([\n    pd.read_csv(\"../input/test_date.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices<1156]+1])).values,\n    pd.read_csv(\"../input/test_numeric.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices>=1156] +1 - 1156])).values\n], axis=1)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a796c10a-d149-0608-350b-6adb5c733e8b"},"outputs":[],"source":"# generate predictions at the chosen threshold\npreds = (clf.predict_proba(X)[:,1] > best_threshold).astype(np.int8)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e0574c4a-ece0-6545-fbc9-cafe21d0844c"},"outputs":[],"source":"# and submit\nsub = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\nsub[\"Response\"] = preds\nsub.to_csv(\"submission.csv.gz\", compression=\"gzip\")"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}