{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.metrics import matthews_corrcoef\nfrom operator import itemgetter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_station_ohe():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    testfile = 'test_date.csv'\n    \n    features = None\n    subset = None\n    train_date_part = pd.read_csv(directory + trainfile, nrows=10000)\n    date_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\n    date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\n    date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\n    stations = list([f.split('_')[1] for f in date_cols ])\n    stations = sorted(stations,key= lambda x: int(x[1:]))\n    \n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          usecols=['Id'] + date_cols,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        \n        if features is None:\n            features = list(chunk.columns)\n            features.remove('Id')\n        \n        chunk.columns = ['Id'] + stations\n        chunk['start_station'] = -1\n        chunk['end_station'] = -1\n        for s in stations:\n            chunk[s] = 1 * (chunk[s] >= 0)\n            id_not_null = chunk[chunk[s] == 1].Id\n            chunk.loc[(chunk['start_station']== -1) & (chunk.Id.isin(id_not_null)),'start_station'] = int(s[1:])\n            chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])   \n        subset = pd.concat([subset, chunk])\n        del chunk\n        gc.collect()\n\n    for i, chunk in enumerate(pd.read_csv(directory + testfile,\n                                          usecols=['Id'] + date_cols,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        #print(i)\n        \n        chunk.columns = ['Id'] + stations\n        chunk['start_station'] = -1\n        chunk['end_station'] = -1\n        for s in stations:\n            chunk[s] = 1 * (chunk[s] >= 0)\n            id_not_null = chunk[chunk[s] == 1].Id\n            chunk.loc[(chunk['start_station']== -1) & (chunk.Id.isin(id_not_null)),'start_station'] = int(s[1:])\n            chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])   \n        subset = pd.concat([subset, chunk])\n        del chunk\n        gc.collect()      \n        \n    return subset,stations,date_cols\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"station_ohe,stations,date_cols = get_station_ohe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"station_ohe['path_len'] = station_ohe[stations].sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"station_ohe.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"station_ohe.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_date_features():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    \n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          chunksize=1,\n                                          low_memory=False)):\n        features = list(chunk.columns)\n        del chunk\n        break\n\n    seen = np.zeros(52)\n    rv = []\n    for f in features:\n        if f == 'Id' or 'S24' in f or 'S25' in f:\n            rv.append(f)\n            continue\n            \n        station = int(f.split('_')[1][1:])\n        \n        if seen[station]:\n            continue\n        \n        seen[station] = 1\n        rv.append(f)\n        \n    return rv\n        \nusefuldatefeatures = get_date_features()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_new_feats():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    testfile = 'test_date.csv'\n    \n    features = None\n    subset = None\n    \n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        #print(i)\n       \n        if features is None:\n            features = list(chunk.columns)\n            features.remove('Id')\n        week_duration = 1679\n        df_mindate_chunk = chunk[['Id']].copy()\n        df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n        df_mindate_chunk['maxdate'] = chunk[features].max(axis=1).values\n        df_mindate_chunk['duration'] =  df_mindate_chunk['maxdate'] - df_mindate_chunk['mindate']\n        df_mindate_chunk['part_week'] = ((df_mindate_chunk['mindate'].values * 100)  % week_duration).astype(np.int64)\n        df_mindate_chunk['min_time_station'] =  chunk[features].idxmin(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n        df_mindate_chunk['max_time_station'] =  chunk[features].idxmax(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n\n        \n        if subset is None:\n            subset = df_mindate_chunk.copy()\n        else:\n            subset = pd.concat([subset, df_mindate_chunk])\n            \n        del chunk\n        gc.collect()\n\n    for i, chunk in enumerate(pd.read_csv(directory + testfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        #print(i)\n        df_mindate_chunk = chunk[['Id']].copy()\n        df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n        df_mindate_chunk['maxdate'] = chunk[features].max(axis=1).values\n        df_mindate_chunk['duration'] =  df_mindate_chunk['maxdate'] - df_mindate_chunk['mindate']\n        df_mindate_chunk['part_week'] = ((df_mindate_chunk['mindate'].values * 100)  % week_duration).astype(np.int64)\n        df_mindate_chunk['min_time_station'] =  chunk[features].idxmin(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n        df_mindate_chunk['max_time_station'] =  chunk[features].idxmax(axis = 1).apply(lambda s: int(s.split('_')[1][1:]) if s is not np.nan else -1)\n        \n        subset = pd.concat([subset, df_mindate_chunk])\n        \n        del chunk\n        gc.collect()    \n        \n        \n    return subset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features = create_new_feats()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features.sort_values(by=['mindate', 'Id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features['mindate_id_diff'] = new_features.Id.diff()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"midr = np.full_like(new_features.mindate_id_diff.values, np.nan)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"midr[0:-1] = -new_features.mindate_id_diff.values[1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features['mindate_id_diff_reverse'] = midr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mcc(tp, tn, fp, fn):\n    sup = tp * tn - fp * fn\n    inf = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if inf == 0:\n        return 0\n    else:\n        return sup / np.sqrt(inf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_mcc(y_true, y_prob, show=False):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true)  # number of positive\n    numn = n - nump  # number of negative\n    tp = nump\n    tn = 0.0\n    fp = numn\n    fn = 0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    if show:\n        best_proba = y_prob[idx[best_id]]\n        y_pred = (y_prob > best_proba).astype(int)\n        return best_proba, best_mcc, y_pred\n    else:\n        return best_mcc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mcc_eval(y_prob, dtrain):\n    y_true = dtrain.get_label()\n    best_mcc = eval_mcc(y_true, y_prob)\n    return 'MCC', best_mcc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_importance(gbm, features):\n    create_feature_map(features)\n    importance = gbm.get_fscore(fmap='xgb.fmap')\n    importance = sorted(importance.items(), key=itemgetter(1), reverse=True)\n    return importance","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"directory = '../input/'\ntrainfiles = ['train_date.csv',\n                  'train_numeric.csv']\ntestfiles = ['test_date.csv',\n                 'test_numeric.csv']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#feature generate from Xgboost with 200,000 records\nnum_feats = ['Id',\n 'L3_S33_F3855',\n 'L3_S32_F3850',\n 'L3_S33_F3865',\n 'L1_S24_F1581',\n 'L3_S38_F3952',\n 'L1_S24_F1672',\n 'L1_S24_F1632',\n 'L1_S24_F1846',\n 'L1_S24_F1844',\n 'L1_S24_F1609',\n 'L1_S24_F1667',\n 'L1_S24_F1842',\n 'L0_S13_F356',\n 'L3_S29_F3342',\n 'L3_S29_F3407',\n 'L3_S34_F3876',\n 'L0_S11_F302',\n 'L3_S29_F3461',\n 'L3_S30_F3494',\n 'L0_S3_F100',\n 'L0_S1_F28',\n 'L0_S6_F122',\n 'L0_S0_F0',\n 'L0_S0_F20',\n 'L3_S30_F3704',\n 'Response']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(date_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [['Id']+date_cols,num_feats]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindata = None\ntestdata = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, f in enumerate(trainfiles):\n        print(f)\n        subset = None\n        for i, chunk in enumerate(pd.read_csv(directory + f,\n                                              usecols=cols[i],\n                                              chunksize=50000,\n                                              low_memory=False)):\n            #print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n        if traindata is None:\n            traindata = subset.copy()\n        else:\n            traindata = pd.merge(traindata, subset.copy(), on=\"Id\")\n        del subset\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del cols[1][-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, f in enumerate(testfiles):\n        print(f)\n        subset = None\n        for i, chunk in enumerate(pd.read_csv(directory + f,\n                                              usecols=cols[i],\n                                              chunksize=50000,\n                                              low_memory=False)):\n            #print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n        if testdata is None:\n            testdata = subset.copy()\n        else:\n            testdata = pd.merge(testdata, subset.copy(), on=\"Id\")\n        del subset\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del midr\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindata = traindata.merge(new_features, on='Id')\ntraindata = traindata.merge(station_ohe, on='Id')\ntestdata = testdata.merge(new_features, on='Id')\ntestdata = testdata.merge(station_ohe, on='Id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del new_features\ndel station_ohe\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testdata['Response'] = 0 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visibletraindata = traindata[::2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"blindtraindata = traindata[1::2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del traindata\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def LeaveOneOut(data1, data2, columnName, useLOO=False):\n    grpOutcomes = data1.groupby(columnName)['Response'].mean().reset_index()\n    grpCount = data1.groupby(columnName)['Response'].count().reset_index()\n    grpOutcomes['cnt'] = grpCount.Response\n    if(useLOO):\n        grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]\n    grpOutcomes.drop('cnt', inplace=True, axis=1)\n    outcomes = data2['Response'].values\n    x = pd.merge(data2[[columnName, 'Response']], grpOutcomes,\n                 suffixes=('x_', ''),\n                 how='left',\n                 on=columnName,\n                 left_index=True)['Response']\n    if(useLOO):\n        x = ((x*x.shape[0])-outcomes)/(x.shape[0]-1)\n        #  x = x + np.random.normal(0, .01, x.shape[0])\n    return x.fillna(x.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(2):\n        for col in cols[i][1:]:\n            blindtraindata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                                     blindtraindata,\n                                                     col, False).values\n            testdata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                               testdata, col, False).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_rounds =52\nparams = {}\nparams['objective'] = \"binary:logistic\"\nparams['eta'] = 0.02105\nparams['max_depth'] = 28\nparams['colsample_bytree'] = 0.999\nparams['subsample'] = 0.999999\nparams['min_child_weight'] = 3\nparams['base_score'] = 0.0044\nparams['silent'] = True\nparams['eval_metric']='auc'\nprint('Fitting')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainpredictions = None\ntestpredictions = None\nfeatures = list(blindtraindata.columns)\nfeatures.remove('Response')\nfeatures.remove('Id')\ndvisibletrain = \\\n        xgb.DMatrix(blindtraindata[features],\n                    blindtraindata.Response,\n                    silent=True)\ndtest = \\\n        xgb.DMatrix(testdata[features],\n                    silent=True)\n\nfolds = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(folds):\n        print('Fold:', i)\n        params['seed'] = i\n        watchlist = [(dvisibletrain, 'train'), (dvisibletrain, 'val')]\n        clf = xgb.train(params, dvisibletrain,\n                        num_boost_round=num_rounds,\n                        evals=watchlist,\n                        early_stopping_rounds=20,\n                        feval=mcc_eval,\n                        maximize=True\n                        \n                        )\n        limit = clf.best_iteration+1\n        predictions = \\\n            clf.predict(dvisibletrain, ntree_limit=limit)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_proba, best_mcc, y_pred = eval_mcc(dvisibletrain.Response,\n                                                predictions,\n                                                True)\nprint(best_proba)\nprint(best_mcc)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_feature_map(features):\n    outfile = open('xgb.fmap', 'w')\n    for i, feat in enumerate(features):\n        outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n    outfile.close()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if(trainpredictions is None):\n            trainpredictions = predictions\nelse:\n            trainpredictions += predictions\npredictions = clf.predict(dtest, ntree_limit=limit)\nif(testpredictions is None):\n            testpredictions = predictions\nelse:\n            testpredictions += predictions\nimp = get_importance(clf, features)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = (testpredictions/folds > 0.4).astype(int)\nsubmission = pd.DataFrame({\"Id\": testdata.Id.values,\n                               \"Response\": y_pred})\nsubmission[['Id', 'Response']].to_csv('xgbsubmission'+str(folds)+'.csv',\n                                          index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, ax = plt.subplots(figsize=(12,18))\nxgb.plot_importance(clf, max_num_features=50, height=0.8, ax=ax,importance_type='gain')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}