{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7f0257e0-628d-7bb5-494e-2877630ef9ae"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\n\nDATA_DIR = \"../input\"\n\nID_COLUMN = 'Id'\nTARGET_COLUMN = 'Response'\n\nSEED = 0\nCHUNKSIZE = 50000\nNROWS = 250000\n\nTRAIN_NUMERIC = \"{0}/train_numeric.csv\".format(DATA_DIR)\nTRAIN_DATE = \"{0}/train_date.csv\".format(DATA_DIR)\n\nTEST_NUMERIC = \"{0}/test_numeric.csv\".format(DATA_DIR)\nTEST_DATE = \"{0}/test_date.csv\".format(DATA_DIR)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"dd4a84b6-044c-c07c-c324-d3db76a7f0fc"},"outputs":[],"source":"train = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\ntest = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n\ntrain[\"StartTime\"] = -1\ntest[\"StartTime\"] = -1"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4bcc0a95-7f19-1d9c-1cef-a328e270b5ba"},"outputs":[],"source":"nrows = 0\nfor tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n    feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n\n    stime_tr = tr[feats].min(axis=1).values\n    stime_te = te[feats].min(axis=1).values\n\n    train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n    test.loc[test.Id.isin(te.Id), 'StartTime'] = stime_te\n\n    nrows += CHUNKSIZE\n    if nrows >= NROWS:\n        break\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"188920b1-6ea8-125c-a63b-09048b8a221d"},"outputs":[],"source":"print (nrows)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8aba9ff6-8f24-de06-49b1-78f4c54994dc"},"outputs":[],"source":"ntrain = train.shape[0]\ntrain_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n\ntrain_test['0_¯\\_(ツ)_/¯_1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['0_¯\\_(ツ)_/¯_2'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n\ntrain_test['0_¯\\_(ツ)_/¯_3'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['0_¯\\_(ツ)_/¯_4'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['index']).drop(['index'], axis=1)\ntrain = train_test.iloc[:ntrain, :]\n\nfeatures = np.setdiff1d(list(train.columns), [TARGET_COLUMN, ID_COLUMN])\n\ny = train.Response.ravel()\ntrain = np.array(train[features])\n\nprint('train: {0}'.format(train.shape))\nprior = np.sum(y) / (1.*len(y))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f0971272-498a-4843-f1c9-fd1efeb1f3d1"},"outputs":[],"source":"train_test\n"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}