{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e5ed31e6-c976-8b9c-ea49-c3eacfbd3b1e"},"outputs":[],"source":"import gc\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.cross_validation import StratifiedKFold\nfrom sklearn.metrics import matthews_corrcoef\nfrom operator import itemgetter\n\n# per raddar, all date features except for stations 24+25 are identical\n\n\n\n\n\ndef get_date_features():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    \n\n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          chunksize=1,\n                                          low_memory=False)):\n        features = list(chunk.columns)\n        break\n\n    seen = np.zeros(52)\n    rv = []\n    for f in features:\n        if f == 'Id':\n            rv.append(f)\n            continue\n            \n        station = int(f.split('_')[1][1:])\n        \n        if seen[station]:\n            continue\n        \n        seen[station] = 1\n        rv.append(f)\n        \n    return rv\n        \nusefuldatefeatures = get_date_features()\n\ndef get_mindate():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    testfile = 'test_date.csv'\n    \n    features = None\n    subset = None\n    \n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        print(i)\n        \n        if features is None:\n            features = list(chunk.columns)\n            features.remove('Id')\n        \n        df_mindate_chunk = chunk[['Id']].copy()\n        df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n        \n        if subset is None:\n            subset = df_mindate_chunk.copy()\n        else:\n            subset = pd.concat([subset, df_mindate_chunk])\n            \n        del chunk\n\n        gc.collect()\n\n    for i, chunk in enumerate(pd.read_csv(directory + testfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        print(i)\n        \n        df_mindate_chunk = chunk[['Id']].copy()\n        df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n        subset = pd.concat([subset, df_mindate_chunk])\n        \n        del chunk\n        gc.collect()      \n        \n    return subset\n\n\ndf_mindate = get_mindate()\n\ndf_mindate.sort_values(by=['mindate', 'Id'], inplace=True)\n\ndf_mindate['mindate_id_diff'] = df_mindate.Id.diff()\n\nmidr = np.full_like(df_mindate.mindate_id_diff.values, np.nan)\nmidr[0:-1] = -df_mindate.mindate_id_diff.values[1:]\n\ndf_mindate['mindate_id_diff_reverse'] = midr\n\n\n\n\ndef mcc(tp, tn, fp, fn):\n    sup = tp * tn - fp * fn\n    inf = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if inf == 0:\n        return 0\n    else:\n        return sup / np.sqrt(inf)\n\n\ndef eval_mcc(y_true, y_prob, show=False):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true)  # number of positive\n    numn = n - nump  # number of negative\n    tp = nump\n    tn = 0.0\n    fp = numn\n    fn = 0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    if show:\n        best_proba = y_prob[idx[best_id]]\n        y_pred = (y_prob > best_proba).astype(int)\n        return best_proba, best_mcc, y_pred\n    else:\n        return best_mcc\n\n\ndef mcc_eval(y_prob, dtrain):\n    y_true = dtrain.get_label()\n    best_mcc = eval_mcc(y_true, y_prob)\n    return 'MCC', best_mcc\n\n\ndef create_feature_map(features):\n    outfile = open('xgb.fmap', 'w')\n    for i, feat in enumerate(features):\n        outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n    outfile.close()\n\n\ndef get_importance(gbm, features):\n    create_feature_map(features)\n    importance = gbm.get_fscore(fmap='xgb.fmap')\n    importance = sorted(importance.items(), key=itemgetter(1), reverse=True)\n    return importance\n\n\ndef LeaveOneOut(data1, data2, columnName, useLOO=False):\n    grpOutcomes = data1.groupby(columnName)['Response'].mean().reset_index()\n    grpCount = data1.groupby(columnName)['Response'].count().reset_index()\n    grpOutcomes['cnt'] = grpCount.Response\n    if(useLOO):\n        grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]\n    grpOutcomes.drop('cnt', inplace=True, axis=1)\n    outcomes = data2['Response'].values\n    x = pd.merge(data2[[columnName, 'Response']], grpOutcomes,\n                 suffixes=('x_', ''),\n                 how='left',\n                 on=columnName,\n                 left_index=True)['Response']\n    if(useLOO):\n        x = ((x*x.shape[0])-outcomes)/(x.shape[0]-1)\n        #  x = x + np.random.normal(0, .01, x.shape[0])\n    return x.fillna(x.mean())\n\n\ndef GrabData():\n    directory = '../input/'\n    trainfiles = ['train_categorical.csv',\n                  'train_date.csv',\n                  'train_numeric.csv']\n    testfiles = ['test_categorical.csv',\n                 'test_date.csv',\n                 'test_numeric.csv']\n\n    cols = [['Id',\n             'L1_S24_F1559', 'L3_S32_F3851',\n             'L1_S24_F1827', 'L1_S24_F1582',\n             'L3_S32_F3854', 'L1_S24_F1510',\n             'L1_S24_F1525', 'L2_S26_F3099'],\n            ['Id',\n             'L3_S30_D3496', 'L3_S30_D3506',\n             'L3_S30_D3501', 'L3_S30_D3516',\n             'L3_S30_D3511', 'L3_S32_D3852',\n             'L3_S33_D3858', 'L3_S34_D3875',\n             'L3_S29_D3316'],\n            ['Id',\n             'L1_S24_F1846', 'L3_S32_F3850',\n             'L1_S24_F1695', 'L1_S24_F1632',\n             'L3_S33_F3855', 'L1_S24_F1604',\n             'L3_S29_F3407', 'L3_S33_F3865',\n             'L3_S38_F3952', 'L1_S24_F1723',\n             'L3_S33_F3861', 'L3_S33_F3857',\n             'L3_S33_F3859', 'L3_S34_F3876', 'L3_S29_F3461',\n             'Response']]\n    traindata = None\n    testdata = None\n\n    for i, f in enumerate(trainfiles):\n        print(f)\n        subset = None\n        for i, chunk in enumerate(pd.read_csv(directory + f,\n                                              usecols=cols[i],\n                                              chunksize=50000,\n                                              low_memory=False)):\n\n            print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n        if traindata is None:\n            traindata = subset.copy()\n        else:\n            traindata = pd.merge(traindata, subset.copy(), on=\"Id\")\n        del subset\n        gc.collect()\n    del cols[2][-1]  # Test doesn't have response!\n    for i, f in enumerate(testfiles):\n        print(f)\n        subset = None\n        for i, chunk in enumerate(pd.read_csv(directory + f,\n                                              usecols=cols[i],\n                                              chunksize=50000,\n                                              low_memory=False)):\n            print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n        if testdata is None:\n            testdata = subset.copy()\n        else:\n            testdata = pd.merge(testdata, subset.copy(), on=\"Id\")\n        del subset\n        gc.collect()\n        \n    traindata = traindata.merge(df_mindate, on='Id')\n    testdata = testdata.merge(df_mindate, on='Id')\n        \n    testdata['Response'] = 0  # Add Dummy Value\n    visibletraindata = traindata[::2]\n    blindtraindata = traindata[1::2]\n    print(blindtraindata.columns)\n    for i in range(2):\n        for col in cols[i][1:]:\n            print(col)\n            blindtraindata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                                     blindtraindata,\n                                                     col, False).values\n            testdata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                               testdata, col, False).values\n    del visibletraindata\n    gc.collect()\n    testdata.drop('Response', inplace=True, axis=1)\n    return blindtraindata, testdata\n\n\ndef Train():\n    train, test = GrabData()\n    print('Train:', train.shape)\n    print('Test', test.shape)\n    features = list(train.columns)\n    features.remove('Response')\n    features.remove('Id')\n    print(features)\n    num_rounds = 50\n    params = {}\n    params['objective'] = \"binary:logistic\"\n    params['eta'] = 0.021\n    params['max_depth'] = 7\n    params['colsample_bytree'] = 0.82\n    params['min_child_weight'] = 3\n    params['base_score'] = 0.005\n    params['silent'] = True\n\n    print('Fitting')\n    trainpredictions = None\n    testpredictions = None\n\n    dvisibletrain = \\\n        xgb.DMatrix(train[features],\n                    train.Response,\n                    silent=True)\n    dtest = \\\n        xgb.DMatrix(test[features],\n                    silent=True)\n\n    folds = 1\n    for i in range(folds):\n        print('Fold:', i)\n        params['seed'] = i\n        watchlist = [(dvisibletrain, 'train'), (dvisibletrain, 'val')]\n        clf = xgb.train(params, dvisibletrain,\n                        num_boost_round=num_rounds,\n                        evals=watchlist,\n                        early_stopping_rounds=20,\n                        feval=mcc_eval,\n                        maximize=True\n                        )\n        limit = clf.best_iteration+1\n        # limit = clf.best_ntree_limit\n        predictions = \\\n            clf.predict(dvisibletrain, ntree_limit=limit)\n\n        best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n                                                predictions,\n                                                True)\n        print('tree limit:', limit)\n        print('mcc:', best_mcc)\n        print(matthews_corrcoef(train.Response,\n                                y_pred))\n        if(trainpredictions is None):\n            trainpredictions = predictions\n        else:\n            trainpredictions += predictions\n        predictions = clf.predict(dtest, ntree_limit=limit)\n        if(testpredictions is None):\n            testpredictions = predictions\n        else:\n            testpredictions += predictions\n        imp = get_importance(clf, features)\n        print('Importance array: ', imp)\n\n    best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n                                            trainpredictions/folds,\n                                            True)\n    print(matthews_corrcoef(train.Response,\n                            y_pred))\n\n    submission = pd.DataFrame({\"Id\": train.Id,\n                               \"Prediction\": trainpredictions/folds,\n                               \"Response\": train.Response})\n    submission[['Id',\n                'Prediction',\n                'Response']].to_csv('rawtrainxgbsubmission'+str(folds)+'.csv',\n                                    index=False)\n\n    submission = pd.DataFrame({\"Id\": test.Id.values,\n                               \"Response\": testpredictions/folds})\n    submission[['Id', 'Response']].to_csv('rawxgbsubmission'+str(folds)+'.csv',\n                                          index=False)\n    y_pred = (testpredictions/folds > .1).astype(int)\n    submission = pd.DataFrame({\"Id\": test.Id.values,\n                               \"Response\": y_pred})\n    submission[['Id', 'Response']].to_csv('xgbsubmission'+str(folds)+'.csv',\n                                          index=False)\n\nif __name__ == \"__main__\":\n    print('Started')\n    Train()\n    print('Finished')\n"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}