{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e7026d10-f130-82c0-0a68-a67cc1e8b942"},"outputs":[],"source":"import gc\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.cross_validation import StratifiedKFold\nfrom sklearn.metrics import matthews_corrcoef\nfrom operator import itemgetter\n\n# per raddar, all date features except for stations 24+25 are identical\n\nNROWS = 20000000000\nCHUNKSIZE = 30000\nDATA_DIR = \"../input/\"\n\n\ndef get_date_features():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    \n\n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          chunksize=1,\n                                          low_memory=False)):\n        features = list(chunk.columns)\n        break\n\n    seen = np.zeros(52)\n    rv = []\n    for f in features:\n        if f == 'Id' or 'S24' in f or 'S25' in f: \n            rv.append(f)\n            continue\n            \n        station = int(f.split('_')[1][1:])\n        \n        if seen[station]:\n            continue\n        \n        seen[station] = 1\n        rv.append(f)\n        \n    return rv\n        \nusefuldatefeatures = get_date_features()\n\ndef get_mindate():\n    directory = '../input/'\n    trainfile = 'train_date.csv'\n    testfile = 'test_date.csv'\n    \n    features = None\n    subset = None\n    \n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        print(i)\n        \n        if features is None:\n            features = list(chunk.columns)\n            features.remove('Id')\n        \n        df_mindate_chunk = chunk[['Id']].copy()\n        df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n        \n        if subset is None:\n            subset = df_mindate_chunk.copy()\n        else:\n            subset = pd.concat([subset, df_mindate_chunk])\n            \n        del chunk\n        del df_mindate_chunk\n        gc.collect()\n\n    for i, chunk in enumerate(pd.read_csv(directory + testfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        print(i)\n        \n        df_mindate_chunk = chunk[['Id']].copy()\n        df_mindate_chunk['mindate'] = chunk[features].min(axis=1).values\n        subset = pd.concat([subset, df_mindate_chunk])\n        \n        del chunk\n        del df_mindate_chunk\n        gc.collect()      \n        \n    return subset\n\n\ndef getdf_mindate():\n    df_mindate = get_mindate()\n    \n    df_mindate.sort_values(by=['mindate', 'Id'], inplace=True)\n    \n    df_mindate['mindate_id_diff'] = df_mindate.Id.diff()\n    \n    midr = np.full_like(df_mindate.mindate_id_diff.values, np.nan)\n    midr[0:-1] = -df_mindate.mindate_id_diff.values[1:]\n    \n    df_mindate['mindate_id_diff_reverse'] = midr\n    del midr\n    return df_mindate\n\n\ndef mcc(tp, tn, fp, fn):\n    sup = tp * tn - fp * fn\n    inf = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if inf == 0:\n        return 0\n    else:\n        return sup / np.sqrt(inf)\n\n\ndef eval_mcc(y_true, y_prob, show=False):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true)  # number of positive\n    numn = n - nump  # number of negative\n    tp = nump\n    tn = 0.0\n    fp = numn\n    fn = 0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    if show:\n        best_proba = y_prob[idx[best_id]]\n        y_pred = (y_prob > best_proba).astype(int)\n        return best_proba, best_mcc, y_pred\n    else:\n        return best_mcc\n\n\ndef mcc_eval(y_prob, dtrain):\n    y_true = dtrain.get_label()\n    best_mcc = eval_mcc(y_true, y_prob)\n    return 'MCC', best_mcc\n\n\ndef create_feature_map(features):\n    outfile = open('xgb.fmap', 'w')\n    for i, feat in enumerate(features):\n        outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n    outfile.close()\n\n\ndef get_importance(gbm, features):\n    create_feature_map(features)\n    importance = gbm.get_fscore(fmap='xgb.fmap')\n    importance = sorted(importance.items(), key=itemgetter(1), reverse=True)\n    return importance\n\n\ndef LeaveOneOut(data1, data2, columnName, useLOO=False):\n    grpOutcomes = data1.groupby(columnName)['Response'].mean().reset_index()\n    grpCount = data1.groupby(columnName)['Response'].count().reset_index()\n    grpOutcomes['cnt'] = grpCount.Response\n    if(useLOO):\n        grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]\n    grpOutcomes.drop('cnt', inplace=True, axis=1)\n    outcomes = data2['Response'].values\n    x = pd.merge(data2[[columnName, 'Response']], grpOutcomes,\n                 suffixes=('x_', ''),\n                 how='left',\n                 on=columnName,\n                 left_index=True)['Response']\n    if(useLOO):\n        x = ((x*x.shape[0])-outcomes)/(x.shape[0]-1)\n        #  x = x + np.random.normal(0, .01, x.shape[0])\n    return x.fillna(x.mean())\n\n\ndef GrabData():\n    directory = '../input/'\n    trainfiles = [DATA_DIR + 'train_categorical.csv',\n                  DATA_DIR + 'train_date.csv',\n                  DATA_DIR + 'train_numeric.csv']\n    testfiles = [DATA_DIR + 'test_categorical.csv',\n                 DATA_DIR + 'test_date.csv',\n                 DATA_DIR + 'test_numeric.csv']\n\n    cols = [['Id',\n             'L1_S24_F1559', 'L3_S32_F3851',\n             'L1_S24_F1827', 'L1_S24_F1582',\n             'L3_S32_F3854', 'L1_S24_F1510',\n             'L1_S24_F1525', 'L2_S26_F3099'],\n            ['Id',\n             'L3_S30_D3496', 'L3_S30_D3506',\n             'L3_S30_D3501', 'L3_S30_D3516',\n             'L3_S30_D3511', 'L3_S32_D3852',\n             'L3_S33_D3858', 'L3_S34_D3875',\n             'L3_S29_D3316'],\n            ['Id',\n             'L1_S24_F1846', 'L3_S32_F3850',\n             'L1_S24_F1695', 'L1_S24_F1632',\n             'L3_S33_F3855', 'L1_S24_F1604',\n             'L3_S29_F3407', 'L3_S33_F3865',\n             'L3_S38_F3952', 'L1_S24_F1723',\n             'L3_S33_F3861', 'L3_S33_F3857',\n             'L3_S33_F3859', 'L3_S34_F3876', 'L3_S29_F3461',\n             'Response']]\n    traindata = None\n    testdata = None\n\n    for i, f in enumerate(trainfiles):\n        print(f)\n        subset = None\n        nrows = 0\n        for i, chunk in enumerate(pd.read_csv(f,\n                                              usecols=cols[i],\n                                              chunksize=CHUNKSIZE,\n                                              low_memory=False)):\n            print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n            \n            nrows += CHUNKSIZE\n            if nrows >= NROWS:\n                print(\"break!!!!!\")\n                print(nrows)\n                break\n            \n        if traindata is None:\n            traindata = subset.copy()\n        else:\n            traindata = pd.merge(traindata, subset.copy(), on=\"Id\")\n        del subset\n        gc.collect()        \n    del cols[3][-1]  # Test doesn't have response!\n    \n    for i, f in enumerate(testfiles):\n        print(f)\n        subset = None\n        nrows = 0\n        for i, chunk in enumerate(pd.read_csv(f,\n                                              usecols=cols[i],\n                                              chunksize=CHUNKSIZE,\n                                              low_memory=False)):\n            print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n            \n            nrows += CHUNKSIZE\n            if nrows >= NROWS:\n                print(\"break!!!!!\")\n                print(nrows)\n                break\n        if testdata is None:\n            testdata = subset.copy()\n        else:\n            testdata = pd.merge(testdata, subset.copy(), on=\"Id\")\n        del subset\n        gc.collect()\n\n        \n    df_mindate = getdf_mindate()\n    traindata = traindata.merge(df_mindate, on='Id')\n    testdata = testdata.merge(df_mindate, on='Id')\n    del df_mindate\n\n        \n    testdata['Response'] = 0  # Add Dummy Value\n    visibletraindata = traindata[::2]\n    blindtraindata = traindata[1::2]\n    print(blindtraindata.columns)\n    for i in range(2):\n        for col in cols[i][1:]:\n            print(col)\n            blindtraindata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                                     blindtraindata,\n                                                     col, False).values\n            testdata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                               testdata, col, False).values\n    del visibletraindata\n    gc.collect()\n    testdata.drop('Response', inplace=True, axis=1)\n    return blindtraindata, testdata\n\n\ndef Train():\n    train, test = GrabData()\n    print('Train:', train.shape)\n    print('Test', test.shape)\n    features = list(train.columns)\n    features.remove('Response')\n    features.remove('Id')\n    print(features)\n    num_rounds = 50\n    params = {}\n    params['objective'] = \"binary:logistic\"\n    params['eta'] = 0.021\n    params['max_depth'] = 7\n    params['colsample_bytree'] = 0.82\n    params['min_child_weight'] = 3\n    params['base_score'] = 0.005\n    params['silent'] = True\n\n    print('Fitting')\n    trainpredictions = None\n    testpredictions = None\n\n    dvisibletrain = \\\n        xgb.DMatrix(train[features],\n                    train.Response,\n                    silent=True)\n    dtest = \\\n        xgb.DMatrix(test[features],\n                    silent=True)\n\n    folds = 1\n    for i in range(folds):\n        print('Fold:', i)\n        params['seed'] = i\n        watchlist = [(dvisibletrain, 'train'), (dvisibletrain, 'val')]\n        clf = xgb.train(params, dvisibletrain,\n                        num_boost_round=num_rounds,\n                        evals=watchlist,\n                        early_stopping_rounds=20,\n                        feval=mcc_eval,\n                        maximize=True\n                        )\n        limit = clf.best_iteration+1\n        # limit = clf.best_ntree_limit\n        predictions = \\\n            clf.predict(dvisibletrain, ntree_limit=limit)\n\n        best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n                                                predictions,\n                                                True)\n        print('tree limit:', limit)\n        print('mcc:', best_mcc)\n        print(matthews_corrcoef(train.Response,\n                                y_pred))\n        if(trainpredictions is None):\n            trainpredictions = predictions\n        else:\n            trainpredictions += predictions\n        predictions = clf.predict(dtest, ntree_limit=limit)\n        if(testpredictions is None):\n            testpredictions = predictions\n        else:\n            testpredictions += predictions\n        imp = get_importance(clf, features)\n        print('Importance array: ', imp)\n\n    best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n                                            trainpredictions/folds,\n                                            True)\n    print(matthews_corrcoef(train.Response,\n                            y_pred))\n\n        \n    pd.DataFrame({\"Id\": test.Id.values, \n    \"Response\": (testpredictions/folds > .2).astype(int)}).to_csv(\n        'xgbsubmission20.csv', index=False)\n        \n    \nif __name__ == \"__main__\":\n    print('Started')\n    \n    ID_COLUMN = 'Id'\n    TARGET_COLUMN = 'Response'\n    TRAIN_NUMERIC = \"{0}train_numeric.csv\".format(DATA_DIR)\n    TRAIN_DATE = \"{0}train_date.csv\".format(DATA_DIR)\n    TEST_NUMERIC = \"{0}test_numeric.csv\".format(DATA_DIR)\n    TEST_DATE = \"{0}test_date.csv\".format(DATA_DIR)\n\n    train = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\n    test = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n\n    train[\"StartTime\"] = -1\n    test[\"StartTime\"] = -1\n        \n    nrows = 0\n    for tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n        feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n    \n        stime_tr = tr[feats].min(axis=1).values\n        stime_te = te[feats].min(axis=1).values\n    \n        train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n \n    \n        nrows += CHUNKSIZE\n        if nrows >= NROWS:\n            break\n\n\n    ntrain = train.shape[0]\n    train_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n    \n    train_test['test_1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n    train_test['test_2'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n    \n    train_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n    \n    train_test['test_3'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n    train_test['test_4'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n    \n    train_test = train_test.sort_values(by=['index']).drop(['index'], axis=1)\n    train = train_test.iloc[:ntrain, :]\n    test = train_test.iloc[ntrain:, :]\n    \n    test.drop(['Response'], axis = 1, inplace = True, errors = 'ignore')\n    \n    del train\n    del test\n    del train_test\n    gc.collect()\n    \n    Train()\n    print('Finished')\n"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}