{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c83bf6cb-22fd-0770-a5fa-ee603e7b39e8" }, "outputs": [], "source": [ "import gc\n", "import numpy as np\n", "import pandas as pd\n", "import xgboost as xgb\n", "from sklearn.cross_validation import StratifiedKFold\n", "from sklearn.metrics import matthews_corrcoef\n", "from operator import itemgetter\n", "\n", "# per raddar, all date features except for stations 24+25 are identical\n", "\n", "\n", "\n", "\n", "\n", "def get_date_features():\n", " directory = '../input/'\n", " trainfile = 'train_date.csv'\n", " \n", "\n", " for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n", " chunksize=1,\n", " low_memory=False)):\n", " features = list(chunk.columns)\n", " break\n", "\n", " seen = np.zeros(52)\n", " rv = []\n", " for f in features:\n", " if f == 'Id':\n", " rv.append(f)\n", " continue\n", " \n", " station = int(f.split('_')[1][1:])\n", " \n", " if seen[station]:\n", " continue\n", " \n", " seen[station] = 1\n", " rv.append(f)\n", " \n", " return rv\n", " \n", "usefuldatefeatures = get_date_features()\n", "\n", "def get_mindate():\n", " directory = '../input/'\n", " trainfile = 'train_date.csv'\n", " testfile = 'test_date.csv'\n", " \n", " features = None\n", " subset = None\n", " \n", " for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n", " usecols=usefuldatefeatures,\n", " chunksize=50000,\n", " low_memory=False)):\n", " print(i)\n", " \n", " if features is None:\n", " features = list(chunk.columns)\n", " features.remove('Id')\n", " \n", " df_mindate_chunk = chunk[['Id']].copy()\n", " df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n", " \n", " if subset is None:\n", " subset = df_mindate_chunk.copy()\n", " else:\n", " subset = pd.concat([subset, df_mindate_chunk])\n", " \n", " del chunk\n", "\n", " gc.collect()\n", "\n", " for i, chunk in enumerate(pd.read_csv(directory + testfile,\n", " usecols=usefuldatefeatures,\n", " chunksize=50000,\n", " low_memory=False)):\n", " print(i)\n", " \n", " df_mindate_chunk = chunk[['Id']].copy()\n", " df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n", " subset = pd.concat([subset, df_mindate_chunk])\n", " \n", " del chunk\n", " gc.collect() \n", " \n", " return subset\n", "\n", "\n", "df_mindate = get_mindate()\n", "\n", "df_mindate.sort_values(by=['mindate', 'Id'], inplace=True)\n", "\n", "df_mindate['mindate_id_diff'] = df_mindate.Id.diff()\n", "\n", "midr = np.full_like(df_mindate.mindate_id_diff.values, np.nan)\n", "midr[0:-1] = -df_mindate.mindate_id_diff.values[1:]\n", "\n", "df_mindate['mindate_id_diff_reverse'] = midr\n", "\n", "\n", "\n", "\n", "def mcc(tp, tn, fp, fn):\n", " sup = tp * tn - fp * fn\n", " inf = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n", " if inf == 0:\n", " return 0\n", " else:\n", " return sup / np.sqrt(inf)\n", "\n", "\n", "def eval_mcc(y_true, y_prob, show=False):\n", " idx = np.argsort(y_prob)\n", " y_true_sort = y_true[idx]\n", " n = y_true.shape[0]\n", " nump = 1.0 * np.sum(y_true) # number of positive\n", " numn = n - nump # number of negative\n", " tp = nump\n", " tn = 0.0\n", " fp = numn\n", " fn = 0.0\n", " best_mcc = 0.0\n", " best_id = -1\n", " mccs = np.zeros(n)\n", " for i in range(n):\n", " if y_true_sort[i] == 1:\n", " tp -= 1.0\n", " fn += 1.0\n", " else:\n", " fp -= 1.0\n", " tn += 1.0\n", " new_mcc = mcc(tp, tn, fp, fn)\n", " mccs[i] = new_mcc\n", " if new_mcc >= best_mcc:\n", " best_mcc = new_mcc\n", " best_id = i\n", " if show:\n", " best_proba = y_prob[idx[best_id]]\n", " y_pred = (y_prob > best_proba).astype(int)\n", " return best_proba, best_mcc, y_pred\n", " else:\n", " return best_mcc\n", "\n", "\n", "def mcc_eval(y_prob, dtrain):\n", " y_true = dtrain.get_label()\n", " best_mcc = eval_mcc(y_true, y_prob)\n", " return 'MCC', best_mcc\n", "\n", "\n", "def create_feature_map(features):\n", " outfile = open('xgb.fmap', 'w')\n", " for i, feat in enumerate(features):\n", " outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n", " outfile.close()\n", "\n", "\n", "def get_importance(gbm, features):\n", " create_feature_map(features)\n", " importance = gbm.get_fscore(fmap='xgb.fmap')\n", " importance = sorted(importance.items(), key=itemgetter(1), reverse=True)\n", " return importance\n", "\n", "\n", "def LeaveOneOut(data1, data2, columnName, useLOO=False):\n", " grpOutcomes = data1.groupby(columnName)['Response'].mean().reset_index()\n", " grpCount = data1.groupby(columnName)['Response'].count().reset_index()\n", " grpOutcomes['cnt'] = grpCount.Response\n", " if(useLOO):\n", " grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]\n", " grpOutcomes.drop('cnt', inplace=True, axis=1)\n", " outcomes = data2['Response'].values\n", " x = pd.merge(data2[[columnName, 'Response']], grpOutcomes,\n", " suffixes=('x_', ''),\n", " how='left',\n", " on=columnName,\n", " left_index=True)['Response']\n", " if(useLOO):\n", " x = ((x*x.shape[0])-outcomes)/(x.shape[0]-1)\n", " # x = x + np.random.normal(0, .01, x.shape[0])\n", " return x.fillna(x.mean())\n", "\n", "\n", "def GrabData():\n", " directory = '../input/'\n", " trainfiles = ['train_categorical.csv',\n", " 'train_date.csv',\n", " 'train_numeric.csv']\n", " testfiles = ['test_categorical.csv',\n", " 'test_date.csv',\n", " 'test_numeric.csv']\n", "\n", " cols = [['Id',\n", " 'L1_S24_F1559', 'L3_S32_F3851',\n", " 'L1_S24_F1827', 'L1_S24_F1582',\n", " 'L3_S32_F3854', 'L1_S24_F1510',\n", " 'L1_S24_F1525', 'L2_S26_F3099'],\n", " ['Id',\n", " 'L3_S30_D3496', 'L3_S30_D3506',\n", " 'L3_S30_D3501', 'L3_S30_D3516',\n", " 'L3_S30_D3511', 'L3_S32_D3852',\n", " 'L3_S33_D3858', 'L3_S34_D3875',\n", " 'L3_S29_D3316'],\n", " ['Id',\n", " 'L1_S24_F1846', 'L3_S32_F3850',\n", " 'L1_S24_F1695', 'L1_S24_F1632',\n", " 'L3_S33_F3855', 'L1_S24_F1604',\n", " 'L3_S29_F3407', 'L3_S33_F3865',\n", " 'L3_S38_F3952', 'L1_S24_F1723',\n", " 'L3_S33_F3861', 'L3_S33_F3857',\n", " 'L3_S33_F3859', 'L3_S34_F3876', 'L3_S29_F3461',\n", " 'Response']]\n", " traindata = None\n", " testdata = None\n", "\n", " for i, f in enumerate(trainfiles):\n", " print(f)\n", " subset = None\n", " for i, chunk in enumerate(pd.read_csv(directory + f,\n", " usecols=cols[i],\n", " chunksize=50000,\n", " low_memory=False)):\n", "\n", " print(i)\n", " if subset is None:\n", " subset = chunk.copy()\n", " else:\n", " subset = pd.concat([subset, chunk])\n", " del chunk\n", " gc.collect()\n", " if traindata is None:\n", " traindata = subset.copy()\n", " else:\n", " traindata = pd.merge(traindata, subset.copy(), on=\"Id\")\n", " del subset\n", " gc.collect()\n", " del cols[2][-1] # Test doesn't have response!\n", " for i, f in enumerate(testfiles):\n", " print(f)\n", " subset = None\n", " for i, chunk in enumerate(pd.read_csv(directory + f,\n", " usecols=cols[i],\n", " chunksize=50000,\n", " low_memory=False)):\n", " print(i)\n", " if subset is None:\n", " subset = chunk.copy()\n", " else:\n", " subset = pd.concat([subset, chunk])\n", " del chunk\n", " gc.collect()\n", " if testdata is None:\n", " testdata = subset.copy()\n", " else:\n", " testdata = pd.merge(testdata, subset.copy(), on=\"Id\")\n", " del subset\n", " gc.collect()\n", " \n", " traindata = traindata.merge(df_mindate, on='Id')\n", " testdata = testdata.merge(df_mindate, on='Id')\n", " \n", " testdata['Response'] = 0 # Add Dummy Value\n", " visibletraindata = traindata[::2]\n", " blindtraindata = traindata[1::2]\n", " print(blindtraindata.columns)\n", " for i in range(2):\n", " for col in cols[i][1:]:\n", " print(col)\n", " blindtraindata.loc[:, col] = LeaveOneOut(visibletraindata,\n", " blindtraindata,\n", " col, False).values\n", " testdata.loc[:, col] = LeaveOneOut(visibletraindata,\n", " testdata, col, False).values\n", " del visibletraindata\n", " gc.collect()\n", " testdata.drop('Response', inplace=True, axis=1)\n", " return blindtraindata, testdata\n", "\n", "\n", "def Train():\n", " train, test = GrabData()\n", " print('Train:', train.shape)\n", " print('Test', test.shape)\n", " features = list(train.columns)\n", " features.remove('Response')\n", " features.remove('Id')\n", " print(features)\n", " num_rounds = 50\n", " params = {}\n", " params['objective'] = \"binary:logistic\"\n", " params['eta'] = 0.021\n", " params['max_depth'] = 7\n", " params['colsample_bytree'] = 0.82\n", " params['min_child_weight'] = 3\n", " params['base_score'] = 0.005\n", " params['silent'] = True\n", "\n", " print('Fitting')\n", " trainpredictions = None\n", " testpredictions = None\n", "\n", " dvisibletrain = \\\n", " xgb.DMatrix(train[features],\n", " train.Response,\n", " silent=True)\n", " dtest = \\\n", " xgb.DMatrix(test[features],\n", " silent=True)\n", "\n", " folds = 1\n", " for i in range(folds):\n", " print('Fold:', i)\n", " params['seed'] = i\n", " watchlist = [(dvisibletrain, 'train'), (dvisibletrain, 'val')]\n", " clf = xgb.train(params, dvisibletrain,\n", " num_boost_round=num_rounds,\n", " evals=watchlist,\n", " early_stopping_rounds=20,\n", " feval=mcc_eval,\n", " maximize=True\n", " )\n", " limit = clf.best_iteration+1\n", " # limit = clf.best_ntree_limit\n", " predictions = \\\n", " clf.predict(dvisibletrain, ntree_limit=limit)\n", "\n", " best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n", " predictions,\n", " True)\n", " print('tree limit:', limit)\n", " print('mcc:', best_mcc)\n", " print(matthews_corrcoef(train.Response,\n", " y_pred))\n", " if(trainpredictions is None):\n", " trainpredictions = predictions\n", " else:\n", " trainpredictions += predictions\n", " predictions = clf.predict(dtest, ntree_limit=limit)\n", " if(testpredictions is None):\n", " testpredictions = predictions\n", " else:\n", " testpredictions += predictions\n", " imp = get_importance(clf, features)\n", " print('Importance array: ', imp)\n", "\n", " best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n", " trainpredictions/folds,\n", " True)\n", " print(matthews_corrcoef(train.Response,\n", " y_pred))\n", "\n", " submission = pd.DataFrame({\"Id\": train.Id,\n", " \"Prediction\": trainpredictions/folds,\n", " \"Response\": train.Response})\n", " submission[['Id',\n", " 'Prediction',\n", " 'Response']].to_csv('rawtrainxgbsubmission'+str(folds)+'.csv',\n", " index=False)\n", "\n", " submission = pd.DataFrame({\"Id\": test.Id.values,\n", " \"Response\": testpredictions/folds})\n", " submission[['Id', 'Response']].to_csv('rawxgbsubmission'+str(folds)+'.csv',\n", " index=False)\n", " y_pred = (testpredictions/folds > .22).astype(int)\n", " submission = pd.DataFrame({\"Id\": test.Id.values,\n", " \"Response\": y_pred})\n", " submission[['Id', 'Response']].to_csv('xgbsubmission'+str(folds)+'.csv',\n", " index=False)\n", "\n", "if __name__ == \"__main__\":\n", " print('Started')\n", " Train()\n", " print('Finished')\n", " " ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0 }