{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Following a similar recipe to lewis' R script (https://www.kaggle.com/cartographic/bosch-production-line-performance/bish-bash-xgboost), sampling the data to select features before running on the full set in order to stay within kaggle's memory limits. Here I add in the train_date data too.\n\nPlease feel free to fork and improve.","metadata":{"_cell_guid":"02eaf587-125d-a4fe-62c7-3dac63b6c75f"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import matthews_corrcoef, roc_auc_score\nfrom sklearn.cross_validation import cross_val_score, StratifiedKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"_cell_guid":"b8ff0786-470f-f3de-e826-2334abc61cda","execution":{"iopub.status.busy":"2021-07-21T14:56:45.374014Z","iopub.execute_input":"2021-07-21T14:56:45.374827Z","iopub.status.idle":"2021-07-21T14:56:46.787748Z","shell.execute_reply.started":"2021-07-21T14:56:45.374661Z","shell.execute_reply":"2021-07-21T14:56:46.785124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I'm limited by RAM here and taking the first N rows is likely to be\n# a bad idea for the date data since it is ordered.\n# Sample the data in a roundabout way:\ndate_chunks = pd.read_csv(\"../input/train_date.csv\", index_col=0, chunksize=100000, dtype=np.float32)\nnum_chunks = pd.read_csv(\"../input/train_numeric.csv\", index_col=0,\n                         usecols=list(range(969)), chunksize=100000, dtype=np.float32)\nX = pd.concat([pd.concat([dchunk, nchunk], axis=1).sample(frac=0.05)\n               for dchunk, nchunk in zip(date_chunks, num_chunks)])\ny = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, usecols=[0,969], dtype=np.float32).loc[X.index].values.ravel()\nX = X.values","metadata":{"_cell_guid":"eb108407-22f9-9716-1905-d6ebfd6a5b59","execution":{"iopub.status.busy":"2021-07-21T14:56:46.788998Z","iopub.status.idle":"2021-07-21T14:56:46.789442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = XGBClassifier(base_score=0.005)\nclf.fit(X, y)","metadata":{"_cell_guid":"7fa0923e-ddfa-d046-f7a8-842667883110","execution":{"iopub.status.busy":"2021-07-21T14:56:46.790723Z","iopub.status.idle":"2021-07-21T14:56:46.791149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# threshold for a manageable number of features\nplt.hist(clf.feature_importances_[clf.feature_importances_>0])\nimportant_indices = np.where(clf.feature_importances_>0.005)[0]\nprint(important_indices)","metadata":{"_cell_guid":"c6159cc6-47b5-e632-b0b6-2f8e0578876d","execution":{"iopub.status.busy":"2021-07-21T14:56:46.792197Z","iopub.status.idle":"2021-07-21T14:56:46.792633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load entire dataset for these features. \n# note where the feature indices are split so we can load the correct ones straight from read_csv\nn_date_features = 1156\nX = np.concatenate([\n    pd.read_csv(\"../input/train_date.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices < n_date_features] + 1])).values,\n    pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices >= n_date_features] + 1 - 1156])).values\n], axis=1)\ny = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32, usecols=[0,969]).values.ravel()","metadata":{"_cell_guid":"d2a8fa06-d180-d316-2f92-aef473339726","execution":{"iopub.status.busy":"2021-07-21T14:56:46.79386Z","iopub.status.idle":"2021-07-21T14:56:46.794272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = XGBClassifier(max_depth=5, base_score=0.005)\ncv = StratifiedKFold(y, n_folds=3)\npreds = np.ones(y.shape[0])\nfor i, (train, test) in enumerate(cv):\n    preds[test] = clf.fit(X[train], y[train]).predict_proba(X[test])[:,1]\n    print(\"fold {}, ROC AUC: {:.3f}\".format(i, roc_auc_score(y[test], preds[test])))\nprint(roc_auc_score(y, preds))","metadata":{"_cell_guid":"0dd2d440-bb20-82c5-7519-d1eb24a7e510","execution":{"iopub.status.busy":"2021-07-21T14:56:46.795187Z","iopub.status.idle":"2021-07-21T14:56:46.7956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pick the best threshold out-of-fold\nthresholds = np.linspace(0.01, 0.99, 50)\nmcc = np.array([matthews_corrcoef(y, preds>thr) for thr in thresholds])\nplt.plot(thresholds, mcc)\nbest_threshold = thresholds[mcc.argmax()]\nprint(mcc.max())","metadata":{"_cell_guid":"eca4bd3d-586a-233d-6dcd-4380939e2b1e","execution":{"iopub.status.busy":"2021-07-21T14:56:46.796445Z","iopub.status.idle":"2021-07-21T14:56:46.796867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load test data\nX = np.concatenate([\n    pd.read_csv(\"../input/test_date.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices<1156]+1])).values,\n    pd.read_csv(\"../input/test_numeric.csv\", index_col=0, dtype=np.float32,\n                usecols=np.concatenate([[0], important_indices[important_indices>=1156] +1 - 1156])).values\n], axis=1)","metadata":{"_cell_guid":"0e9aceeb-3b06-74af-3238-c500a33d5a71","execution":{"iopub.status.busy":"2021-07-21T14:56:46.797744Z","iopub.status.idle":"2021-07-21T14:56:46.798141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate predictions at the chosen threshold\npreds = (clf.predict_proba(X)[:,1] > best_threshold).astype(np.int8)","metadata":{"_cell_guid":"a796c10a-d149-0608-350b-6adb5c733e8b","execution":{"iopub.status.busy":"2021-07-21T14:56:46.799016Z","iopub.status.idle":"2021-07-21T14:56:46.799416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# and submit\nsub = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\nsub[\"Response\"] = preds\nsub.to_csv(\"submission.csv.gz\", compression=\"gzip\")","metadata":{"_cell_guid":"e0574c4a-ece0-6545-fbc9-cafe21d0844c","execution":{"iopub.status.busy":"2021-07-21T14:56:46.800268Z","iopub.status.idle":"2021-07-21T14:56:46.800712Z"},"trusted":true},"execution_count":null,"outputs":[]}]}