{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.auto import tqdm\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras import optimizers, Sequential, Model\nimport keras","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:04.822943Z","iopub.execute_input":"2021-09-02T18:44:04.823639Z","iopub.status.idle":"2021-09-02T18:44:12.734677Z","shell.execute_reply.started":"2021-09-02T18:44:04.823533Z","shell.execute_reply":"2021-09-02T18:44:12.733571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/beyond-analysis/train.csv')\ntrain_df.drop(columns = ['CATEGORY_1', 'CATEGORY_2'], axis=0, inplace = True)\n\nusers = train_df['UNIQUE_IDENTIFIER'].unique()\nusers_train, users_test = train_test_split(users, test_size=0.2, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:12.738053Z","iopub.execute_input":"2021-09-02T18:44:12.738579Z","iopub.status.idle":"2021-09-02T18:44:20.166166Z","shell.execute_reply.started":"2021-09-02T18:44:12.738531Z","shell.execute_reply":"2021-09-02T18:44:20.164783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nf = open('../input/techniche-lstm-version-3-data/x_train.json')\nX = json.load(f)\nf = open('../input/techniche-lstm-version-3-data/y_train.json')\nY = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:20.168162Z","iopub.execute_input":"2021-09-02T18:44:20.168526Z","iopub.status.idle":"2021-09-02T18:44:30.952460Z","shell.execute_reply.started":"2021-09-02T18:44:20.168460Z","shell.execute_reply":"2021-09-02T18:44:30.951139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['UNIQUE_IDENTIFIER']== users[0]]","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:30.953931Z","iopub.execute_input":"2021-09-02T18:44:30.954236Z","iopub.status.idle":"2021-09-02T18:44:31.045020Z","shell.execute_reply.started":"2021-09-02T18:44:30.954206Z","shell.execute_reply":"2021-09-02T18:44:31.043746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad(users, X=X, Y=Y, pad_size=30):\n    final_x = []\n    final_y1 = []\n    final_y2 = []\n    for user_id in tqdm(users):\n        x = np.array(X[str(user_id)]).T\n        y = np.array(Y[str(user_id)]).T\n\n        pad_array = np.zeros((x.shape[0], pad_size - x.shape[1]))\n        x = np.concatenate((x, pad_array), axis=1)\n        final_x.append(x)\n        final_y1.append(y[0])\n        final_y2.append(y[1])\n    final_x, final_y1, final_y2 = np.array(final_x), np.array(final_y1), np.array(final_y2)\n    final_x = np.transpose(final_x, [0, 2, 1])\n    return final_x, final_y1, final_y2","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:47.762828Z","iopub.execute_input":"2021-09-02T18:44:47.763255Z","iopub.status.idle":"2021-09-02T18:44:47.773240Z","shell.execute_reply.started":"2021-09-02T18:44:47.763222Z","shell.execute_reply":"2021-09-02T18:44:47.771755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, train_y1, train_y2 = pad(users_train)\ntest_x, test_y1, test_y2 = pad(users_test)\n#  Remove columns 5, 9, 10, 11, 12. 15, 16, 17, 18, \n","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:52.111606Z","iopub.execute_input":"2021-09-02T18:44:52.112126Z","iopub.status.idle":"2021-09-02T18:44:59.679870Z","shell.execute_reply.started":"2021-09-02T18:44:52.112092Z","shell.execute_reply":"2021-09-02T18:44:59.678405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x.shape, test_x.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:44:59.681441Z","iopub.execute_input":"2021-09-02T18:44:59.681835Z","iopub.status.idle":"2021-09-02T18:44:59.691760Z","shell.execute_reply.started":"2021-09-02T18:44:59.681802Z","shell.execute_reply":"2021-09-02T18:44:59.690239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We removed some features having more than 80% zeros. Also, we do not have UNIQUE_IDENTIFIER and SEQUENCE_NO in our X and Y vectors for each user.","metadata":{}},{"cell_type":"code","source":"cols = list(train_df.columns)\ni = 2\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 3\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 4\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 5\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 6\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 7\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 8\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 9\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 10\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 11\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 12\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 13\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 14\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 15\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 16\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 17\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 18\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 19\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 20\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:45:11.852988Z","iopub.execute_input":"2021-09-02T18:45:11.853352Z","iopub.status.idle":"2021-09-02T18:45:12.567975Z","shell.execute_reply.started":"2021-09-02T18:45:11.853320Z","shell.execute_reply":"2021-09-02T18:45:12.566533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = train_x[..., :5]\nb = train_x[..., 6:9]\nc = train_x[..., 13:15]\ntrain_x_cat = np.concatenate((a, b, c), axis=-1)\ntrain_x_cat.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:45:19.252376Z","iopub.execute_input":"2021-09-02T18:45:19.252771Z","iopub.status.idle":"2021-09-02T18:45:19.430331Z","shell.execute_reply.started":"2021-09-02T18:45:19.252739Z","shell.execute_reply":"2021-09-02T18:45:19.428875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = test_x[..., :5]\nb = test_x[..., 6:9]\nc = test_x[..., 13:15]\ntest_x_cat = np.concatenate((a, b, c), axis=-1)\ntest_x_cat.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:45:25.942143Z","iopub.execute_input":"2021-09-02T18:45:25.942855Z","iopub.status.idle":"2021-09-02T18:45:25.989439Z","shell.execute_reply.started":"2021-09-02T18:45:25.942800Z","shell.execute_reply":"2021-09-02T18:45:25.987994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Y1","metadata":{}},{"cell_type":"code","source":"serie_size =  train_x_cat.shape[1] # 30\nn_features =  train_x_cat.shape[2] # 10\n\nlstm_model_y1 = Sequential()\nlstm_model_y1.add(L.LSTM(10, input_shape=(serie_size, n_features), return_sequences=True))\nlstm_model_y1.add(L.LSTM(6, activation='relu', return_sequences=True))\nlstm_model_y1.add(L.LSTM(1, activation='relu'))\nlstm_model_y1.add(L.Dense(10, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y1.add(L.Dense(10, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y1.add(L.Dense(1))\nlstm_model_y1.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:45:39.364431Z","iopub.execute_input":"2021-09-02T18:45:39.364991Z","iopub.status.idle":"2021-09-02T18:45:40.086114Z","shell.execute_reply.started":"2021-09-02T18:45:39.364954Z","shell.execute_reply":"2021-09-02T18:45:40.084779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2\nbatch = 128*16*4*4\nlr = 0.001\n\nadam = optimizers.Adam(lr)\nlstm_model_y1.compile(loss='mse', optimizer=adam)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:45:46.243698Z","iopub.execute_input":"2021-09-02T18:45:46.244276Z","iopub.status.idle":"2021-09-02T18:45:46.269246Z","shell.execute_reply.started":"2021-09-02T18:45:46.244233Z","shell.execute_reply":"2021-09-02T18:45:46.267818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_history = lstm_model_y1.fit(train_x_cat, train_y1,\n                              validation_data=(test_x_cat, test_y1), \n                              batch_size=batch, \n                              epochs=epochs,\n                              verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:46:22.367016Z","iopub.execute_input":"2021-09-02T18:46:22.367431Z","iopub.status.idle":"2021-09-02T18:46:33.278817Z","shell.execute_reply.started":"2021-09-02T18:46:22.367397Z","shell.execute_reply":"2021-09-02T18:46:33.277600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Y2","metadata":{}},{"cell_type":"code","source":"serie_size =  train_x_cat.shape[1] # 30\nn_features =  train_x_cat.shape[2] # 10\n\nlstm_model_y2 = Sequential()\nlstm_model_y2.add(L.LSTM(10, input_shape=(serie_size, n_features), return_sequences=True))\nlstm_model_y2.add(L.LSTM(6, activation='relu', return_sequences=True))\nlstm_model_y2.add(L.LSTM(1, activation='relu', return_sequences=True))\nlstm_model_y2.add(L.Flatten())\nlstm_model_y2.add(L.Dense(100, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y2.add(L.Dense(100, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y2.add(L.Dense(50, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y2.add(L.Dense(10, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y2.add(L.Dense(1))\nlstm_model_y2.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:46:49.183181Z","iopub.execute_input":"2021-09-02T18:46:49.183588Z","iopub.status.idle":"2021-09-02T18:46:49.750667Z","shell.execute_reply.started":"2021-09-02T18:46:49.183555Z","shell.execute_reply":"2021-09-02T18:46:49.749503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2\nbatch = 128*16*4*4\n\nlr = 0.0001\n\nadam = optimizers.Adam(lr)\nlstm_model_y2.compile(loss='mse', optimizer=adam)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:46:58.122897Z","iopub.execute_input":"2021-09-02T18:46:58.123301Z","iopub.status.idle":"2021-09-02T18:46:58.140278Z","shell.execute_reply.started":"2021-09-02T18:46:58.123268Z","shell.execute_reply":"2021-09-02T18:46:58.139190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_history = lstm_model_y2.fit(train_x_cat, train_y2,\n                              validation_data=(test_x_cat, test_y2), \n                              batch_size=batch, \n                              epochs=epochs,\n                              verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:47:07.467091Z","iopub.execute_input":"2021-09-02T18:47:07.467471Z","iopub.status.idle":"2021-09-02T18:47:17.113834Z","shell.execute_reply.started":"2021-09-02T18:47:07.467439Z","shell.execute_reply":"2021-09-02T18:47:17.112742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading our pretrained models","metadata":{}},{"cell_type":"code","source":"lstm_model_y1 = keras.models.load_model(\"../input/techniche-beyond-analysis-models/1_models/Y1_model.h5\")\nlstm_model_y2 = keras.models.load_model(\"../input/techniche-beyond-analysis-models/1_models/Y2_model.h5\")","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:47:36.433365Z","iopub.execute_input":"2021-09-02T18:47:36.433983Z","iopub.status.idle":"2021-09-02T18:47:37.587368Z","shell.execute_reply.started":"2021-09-02T18:47:36.433951Z","shell.execute_reply":"2021-09-02T18:47:37.585675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y1.predict(train_x_cat[:2]), train_y2[:2]","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:47:46.847550Z","iopub.execute_input":"2021-09-02T18:47:46.847946Z","iopub.status.idle":"2021-09-02T18:47:47.685211Z","shell.execute_reply.started":"2021-09-02T18:47:46.847909Z","shell.execute_reply":"2021-09-02T18:47:47.684047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y2.predict(train_x_cat[:2]), train_y2[:2]","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:47:47.926931Z","iopub.execute_input":"2021-09-02T18:47:47.927311Z","iopub.status.idle":"2021-09-02T18:47:48.695132Z","shell.execute_reply.started":"2021-09-02T18:47:47.927279Z","shell.execute_reply":"2021-09-02T18:47:48.694116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"import json\nf = open('../input/techniche-lstm-version-3-data/x_test.json')\nX_test = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:03.317738Z","iopub.execute_input":"2021-09-02T18:48:03.318366Z","iopub.status.idle":"2021-09-02T18:48:11.279694Z","shell.execute_reply.started":"2021-09-02T18:48:03.318331Z","shell.execute_reply":"2021-09-02T18:48:11.278359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_pad(users, X=X_test, pad_size=30):\n    final_x = []\n    for user_id in tqdm(users):\n        x = np.array(X[str(user_id)]).T\n        pad_array = np.zeros((x.shape[0], pad_size - x.shape[1]))\n        x = np.concatenate((x, pad_array), axis=1)\n        final_x.append(x)\n    final_x = np.array(final_x)\n    final_x = np.transpose(final_x, [0, 2, 1])\n    return final_x","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:11.281926Z","iopub.execute_input":"2021-09-02T18:48:11.282430Z","iopub.status.idle":"2021-09-02T18:48:11.291780Z","shell.execute_reply.started":"2021-09-02T18:48:11.282381Z","shell.execute_reply":"2021-09-02T18:48:11.290223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/beyond-analysis/test.csv')\ntest_df.drop(columns = ['CATEGORY_1', 'CATEGORY_2'], axis=0, inplace = True)\n\nusers_sub = test_df['UNIQUE_IDENTIFIER'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:12.681632Z","iopub.execute_input":"2021-09-02T18:48:12.681995Z","iopub.status.idle":"2021-09-02T18:48:15.969919Z","shell.execute_reply.started":"2021-09-02T18:48:12.681963Z","shell.execute_reply":"2021-09-02T18:48:15.968669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_x = test_pad(users_sub)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:22.717160Z","iopub.execute_input":"2021-09-02T18:48:22.717645Z","iopub.status.idle":"2021-09-02T18:48:27.770976Z","shell.execute_reply.started":"2021-09-02T18:48:22.717610Z","shell.execute_reply":"2021-09-02T18:48:27.769682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_x.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:28.198940Z","iopub.execute_input":"2021-09-02T18:48:28.199392Z","iopub.status.idle":"2021-09-02T18:48:28.207004Z","shell.execute_reply.started":"2021-09-02T18:48:28.199359Z","shell.execute_reply":"2021-09-02T18:48:28.205697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = sub_x[..., :5]\nb = sub_x[..., 6:9]\nc = sub_x[..., 13:15]\nsub_x_cat = np.concatenate((a, b, c), axis=-1)\nsub_x_cat.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:33.467211Z","iopub.execute_input":"2021-09-02T18:48:33.467645Z","iopub.status.idle":"2021-09-02T18:48:33.637197Z","shell.execute_reply.started":"2021-09-02T18:48:33.467607Z","shell.execute_reply":"2021-09-02T18:48:33.635900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y1 = lstm_model_y1.predict(sub_x_cat)\nprint(pred_y1)\npred_y1.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:39.427136Z","iopub.execute_input":"2021-09-02T18:48:39.427497Z","iopub.status.idle":"2021-09-02T18:48:55.006515Z","shell.execute_reply.started":"2021-09-02T18:48:39.427450Z","shell.execute_reply":"2021-09-02T18:48:55.004993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y2 = lstm_model_y2.predict(sub_x_cat)\nprint(pred_y2)\npred_y2.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:48:55.008663Z","iopub.execute_input":"2021-09-02T18:48:55.009088Z","iopub.status.idle":"2021-09-02T18:49:10.747737Z","shell.execute_reply.started":"2021-09-02T18:48:55.009048Z","shell.execute_reply":"2021-09-02T18:49:10.745999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub = pd.DataFrame({'UNIQUE_IDENTIFIER': users_sub.astype(int), \n#                    'Y1': pred_y1[:, 0], 'Y2': pred_y2[:, 0]})\n\n# sub.to_csv('sub_checking.csv', index=False)\nsub = pd.DataFrame({'UNIQUE_IDENTIFIER': users_sub.astype(int), \n                   'Y2': pred_y2[:, 0]})\n\nsub.to_csv('sub_checking.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T18:49:10.750180Z","iopub.execute_input":"2021-09-02T18:49:10.750544Z","iopub.status.idle":"2021-09-02T18:49:10.962887Z","shell.execute_reply.started":"2021-09-02T18:49:10.750511Z","shell.execute_reply":"2021-09-02T18:49:10.961643Z"},"trusted":true},"execution_count":null,"outputs":[]}]}