{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ebcab444-1c71-c57b-d979-83cb17f4e6a8"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\n\nDATA_DIR = \"../input\"\n\nID_COLUMN = 'Id'\nTARGET_COLUMN = 'Response'\n\nSEED = 0\nCHUNKSIZE = 50000\nNROWS = 250000\n\nTRAIN_NUMERIC = \"{0}/train_numeric.csv\".format(DATA_DIR)\nTRAIN_DATE = \"{0}/train_date.csv\".format(DATA_DIR)\n\nTEST_NUMERIC = \"{0}/test_numeric.csv\".format(DATA_DIR)\nTEST_DATE = \"{0}/test_date.csv\".format(DATA_DIR)\n\nFILENAME = \"etimelhoods\"\n\ntrain = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\ntest = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n\ntrain[\"StartTime\"] = -1\ntest[\"StartTime\"] = -1\n\n\nnrows = 0\nfor tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n    feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n\n    stime_tr = tr[feats].min(axis=1).values\n    stime_te = te[feats].min(axis=1).values\n\n    train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n    test.loc[test.Id.isin(te.Id), 'StartTime'] = stime_te\n\n    nrows += CHUNKSIZE\n    if nrows >= NROWS:\n        break\n\n\nntrain = train.shape[0]\ntrain_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"deaa5c6a-e59d-a82f-142d-e25c30eabd0b"},"outputs":[],"source":"train_test.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1571e443-5e9b-aaca-acb6-961cc64341ff"},"outputs":[],"source":"train_test['0_¯\\_(ツ)_/¯_1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f59d04f3-c898-0789-029f-08793f89ac3e"},"outputs":[],"source":"train_test.head()\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2d4c85bf-fd59-9a7d-7c14-18c8c74e3102"},"outputs":[],"source":"train_test['0_¯\\_(ツ)_/¯_2'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7ac413ee-f426-4936-098a-c5ad91e91ef0"},"outputs":[],"source":"train_test.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a5014926-229c-8e74-627b-666dbe98a4b9"},"outputs":[],"source":"train_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n\ntrain_test['0_¯\\_(ツ)_/¯_3'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['0_¯\\_(ツ)_/¯_4'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2e66a952-2bc8-2292-b1fc-6ce848c50da7"},"outputs":[],"source":"train_test.head(10)\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c8c38ca7-580a-3c1b-f54b-9395ed951c44"},"outputs":[],"source":"min(train_test['StartTime'])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"af59a899-4458-5afe-aaee-3c5b2e6ab1d8"},"outputs":[],"source":"train.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"62a8e265-0fcb-4e6c-aa2f-8117b5254fb2"},"outputs":[],"source":"min(train['StartTime'])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"466c9724-1834-800b-3bcc-941d974320a3"},"outputs":[],"source":"ntrain"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"67cf2012-2594-a9e2-084d-69ff6f0c5682"},"outputs":[],"source":"train.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"fb517a8f-a469-8346-1d73-32c0602a405a"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}