{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:34.177049Z","iopub.execute_input":"2021-09-01T14:52:34.177445Z","iopub.status.idle":"2021-09-01T14:52:35.317737Z","shell.execute_reply.started":"2021-09-01T14:52:34.17735Z","shell.execute_reply":"2021-09-01T14:52:35.316688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install catboost","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:35.319243Z","iopub.execute_input":"2021-09-01T14:52:35.319558Z","iopub.status.idle":"2021-09-01T14:52:42.807056Z","shell.execute_reply.started":"2021-09-01T14:52:35.319524Z","shell.execute_reply":"2021-09-01T14:52:42.806133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostRegressor","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:42.810909Z","iopub.execute_input":"2021-09-01T14:52:42.811196Z","iopub.status.idle":"2021-09-01T14:52:42.975314Z","shell.execute_reply.started":"2021-09-01T14:52:42.811166Z","shell.execute_reply":"2021-09-01T14:52:42.974459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = pd.read_csv('../input/beyond-analysis/train.csv')\n# train_df.drop(columns = ['CATEGORY_1', 'CATEGORY_2'], axis=0, inplace = True)\n# train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:42.976886Z","iopub.execute_input":"2021-09-01T14:52:42.977265Z","iopub.status.idle":"2021-09-01T14:52:42.982989Z","shell.execute_reply.started":"2021-09-01T14:52:42.977227Z","shell.execute_reply":"2021-09-01T14:52:42.982282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# users = train_df['UNIQUE_IDENTIFIER'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:42.984419Z","iopub.execute_input":"2021-09-01T14:52:42.984779Z","iopub.status.idle":"2021-09-01T14:52:42.990841Z","shell.execute_reply.started":"2021-09-01T14:52:42.984743Z","shell.execute_reply":"2021-09-01T14:52:42.989992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_i = train_df[train_df['UNIQUE_IDENTIFIER'] == users[0]]\n# x_i = train_i.loc[:,'STATUS_CHECK': 'PRACTICE_WINNINGS_NUMBER']\n# y_i = train_i.loc[:,'Y1': 'Y2']\n# np.array(y_i)[0]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:42.992085Z","iopub.execute_input":"2021-09-01T14:52:42.992479Z","iopub.status.idle":"2021-09-01T14:52:42.99779Z","shell.execute_reply.started":"2021-09-01T14:52:42.992452Z","shell.execute_reply":"2021-09-01T14:52:42.996583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x = {}\n# y = {}\n\n# for i in tqdm(users):\n#     train_i = train_df[train_df['UNIQUE_IDENTIFIER'] == i]\n#     x_i = train_i.loc[:,'STATUS_CHECK': 'PRACTICE_WINNINGS_NUMBER']\n#     y_i = train_i.loc[:,'Y1': 'Y2']\n#     x[i] = np.array(x_i)\n#     y[i] = np.array(y_i)[0]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:42.999451Z","iopub.execute_input":"2021-09-01T14:52:42.99982Z","iopub.status.idle":"2021-09-01T14:52:43.005578Z","shell.execute_reply.started":"2021-09-01T14:52:42.999787Z","shell.execute_reply":"2021-09-01T14:52:43.004739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = {}\n# Y = {}\n# for k in tqdm(x.keys()):\n#     X[str(k)] = x[k].tolist()\n#     Y[str(k)] = y[k].tolist()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.008723Z","iopub.execute_input":"2021-09-01T14:52:43.009102Z","iopub.status.idle":"2021-09-01T14:52:43.014461Z","shell.execute_reply.started":"2021-09-01T14:52:43.009072Z","shell.execute_reply":"2021-09-01T14:52:43.01368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import json\n# with open('./' + 'x_train.json', 'w') as fp:\n#     json.dump(X, fp)\n# with open('./' + 'y_train.json', 'w') as fp:\n#     json.dump(Y, fp)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.015966Z","iopub.execute_input":"2021-09-01T14:52:43.016404Z","iopub.status.idle":"2021-09-01T14:52:43.023344Z","shell.execute_reply.started":"2021-09-01T14:52:43.01637Z","shell.execute_reply":"2021-09-01T14:52:43.022482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.read_csv('../input/beyond-analysis/test.csv')\n# test_df.drop(columns = ['CATEGORY_1', 'CATEGORY_2'], axis=0, inplace = True)\n# users_test = test_df['UNIQUE_IDENTIFIER'].unique()\n# test_df.head()\n\n# x = {}\n\n# for i in tqdm(users_test):\n#     test_i = test_df[test_df['UNIQUE_IDENTIFIER'] == i]\n#     x_i = test_i.loc[:,'STATUS_CHECK': 'PRACTICE_WINNINGS_NUMBER']\n#     x[i] = np.array(x_i)\n    \n# test_X = {}\n# for k in tqdm(x.keys()):\n#     test_X[str(k)] = x[k].tolist()\n\n# import json\n# with open('./' + 'x_test.json', 'w') as fp:\n#     json.dump(test_X, fp)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.024367Z","iopub.execute_input":"2021-09-01T14:52:43.0246Z","iopub.status.idle":"2021-09-01T14:52:43.031497Z","shell.execute_reply.started":"2021-09-01T14:52:43.024578Z","shell.execute_reply":"2021-09-01T14:52:43.030657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import glob\n# import random\n# import os\n# import numpy as np\n\n# import torch\n# from torch.utils.data import Dataset\n# from PIL import Image\n# import torchvision.transforms as transforms\n\n# import torch\n# from torch import nn\n# from matplotlib import pyplot as plt\n\n# import argparse\n# import os\n# import numpy as np\n# import math\n# import itertools\n# import sys\n\n# import torchvision.transforms as transforms\n# from torchvision.utils import save_image, make_grid\n\n# from torch.utils.data import DataLoader\n# from torch.autograd import Variable\n\n# import torch.nn as nn\n# import torch.nn.functional as F\n# import torch","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.032625Z","iopub.execute_input":"2021-09-01T14:52:43.032859Z","iopub.status.idle":"2021-09-01T14:52:43.038911Z","shell.execute_reply.started":"2021-09-01T14:52:43.032837Z","shell.execute_reply":"2021-09-01T14:52:43.038108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pad_length = 15","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.03994Z","iopub.execute_input":"2021-09-01T14:52:43.040367Z","iopub.status.idle":"2021-09-01T14:52:43.05083Z","shell.execute_reply.started":"2021-09-01T14:52:43.040333Z","shell.execute_reply":"2021-09-01T14:52:43.049871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import json\n# f = open('../input/techniche-sequential/x_train.json')\n# X = json.load(f)\n# f = open('../input/techniche-sequential/y_train.json')\n# Y = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.052351Z","iopub.execute_input":"2021-09-01T14:52:43.052595Z","iopub.status.idle":"2021-09-01T14:52:43.058819Z","shell.execute_reply.started":"2021-09-01T14:52:43.052572Z","shell.execute_reply":"2021-09-01T14:52:43.057924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# users_train, users_test = train_test_split(users, test_size=0.2, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.060086Z","iopub.execute_input":"2021-09-01T14:52:43.060329Z","iopub.status.idle":"2021-09-01T14:52:43.066319Z","shell.execute_reply.started":"2021-09-01T14:52:43.060307Z","shell.execute_reply":"2021-09-01T14:52:43.065398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Dataset(Dataset):\n#     def __init__(self, users, X, Y, pad_size = 30):\n        \n#         self.users= users\n#         self.pad_size = pad_size\n#         self.X= X\n#         self.Y= Y\n        \n#     def __getitem__(self, index):\n#         user_id = self.users[index]\n#         x = np.array(self.X[str(user_id)])\n#         y = np.array(self.Y[str(user_id)])\n        \n#         pad_array = np.zeros((x.shape[0], self.pad_size - x.shape[1]))\n#         x = np.concatenate((x, pad_array), axis=1)\n        \n#         x = torch.tensor(x)\n#         y = torch.tensor(y)\n\n#         return {'x': x, 'y' : y}\n\n\n#     def __len__(self):\n#         return len(self.X)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.067429Z","iopub.execute_input":"2021-09-01T14:52:43.067909Z","iopub.status.idle":"2021-09-01T14:52:43.074269Z","shell.execute_reply.started":"2021-09-01T14:52:43.067802Z","shell.execute_reply":"2021-09-01T14:52:43.073431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = Dataset(users_train, X, Y)\n# train = Dataset(users_test, X, Y)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.075381Z","iopub.execute_input":"2021-09-01T14:52:43.075801Z","iopub.status.idle":"2021-09-01T14:52:43.082289Z","shell.execute_reply.started":"2021-09-01T14:52:43.075775Z","shell.execute_reply":"2021-09-01T14:52:43.081472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test[0]['x'].shape, test[0]['y']","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.083595Z","iopub.execute_input":"2021-09-01T14:52:43.084042Z","iopub.status.idle":"2021-09-01T14:52:43.089863Z","shell.execute_reply.started":"2021-09-01T14:52:43.083988Z","shell.execute_reply":"2021-09-01T14:52:43.089092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras import optimizers, Sequential, Model\nimport keras","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:43.091323Z","iopub.execute_input":"2021-09-01T14:52:43.091719Z","iopub.status.idle":"2021-09-01T14:52:47.396653Z","shell.execute_reply.started":"2021-09-01T14:52:43.091681Z","shell.execute_reply":"2021-09-01T14:52:47.395766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/beyond-analysis/train.csv')\ntrain_df.drop(columns = ['CATEGORY_1', 'CATEGORY_2'], axis=0, inplace = True)\n\nusers = train_df['UNIQUE_IDENTIFIER'].unique()\nusers_train, users_test = train_test_split(users, test_size=0.2, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:47.39816Z","iopub.execute_input":"2021-09-01T14:52:47.398547Z","iopub.status.idle":"2021-09-01T14:52:52.151396Z","shell.execute_reply.started":"2021-09-01T14:52:47.398507Z","shell.execute_reply":"2021-09-01T14:52:52.150499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nf = open('../input/techniche-lstm-version-3-data/x_train.json')\nX = json.load(f)\nf = open('../input/techniche-lstm-version-3-data/y_train.json')\nY = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:52:52.152687Z","iopub.execute_input":"2021-09-01T14:52:52.153054Z","iopub.status.idle":"2021-09-01T14:53:00.697894Z","shell.execute_reply.started":"2021-09-01T14:52:52.153004Z","shell.execute_reply":"2021-09-01T14:53:00.697065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['UNIQUE_IDENTIFIER']== users[0]]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:00.699177Z","iopub.execute_input":"2021-09-01T14:53:00.699505Z","iopub.status.idle":"2021-09-01T14:53:00.758772Z","shell.execute_reply.started":"2021-09-01T14:53:00.69947Z","shell.execute_reply":"2021-09-01T14:53:00.757931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = list(train_df.columns)\n#i = 1\n#print(cols[i], \" => \", train_df[cols[i]].value_counts()[0])\ni = 2\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 3\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 4\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 5\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 6\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 7\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 8\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 9\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 10\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 11\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 12\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 13\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 14\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 15\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 16\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 17\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 18\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 19\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))\ni = 20\nprint(cols[i], \" => \", train_df[cols[i]].value_counts()[0]/len(train_df))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:00.76022Z","iopub.execute_input":"2021-09-01T14:53:00.760589Z","iopub.status.idle":"2021-09-01T14:53:01.808128Z","shell.execute_reply.started":"2021-09-01T14:53:00.760549Z","shell.execute_reply":"2021-09-01T14:53:01.807293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pad(users, X=X, Y=Y, pad_size=30):\n    final_x = []\n    final_y1 = []\n    final_y2 = []\n    for user_id in tqdm(users):\n        x = np.array(X[str(user_id)]).T\n        y = np.array(Y[str(user_id)]).T\n        if(x.shape[1]<=pad_size):\n            pad_array = np.zeros((x.shape[0], pad_size - x.shape[1]))\n            x = np.concatenate((x, pad_array), axis=1)\n        else:\n            x = x[:,:pad_size]\n        final_x.append(x)\n        final_y1.append(y[0])\n        final_y2.append(y[1])\n    final_x, final_y1, final_y2 = np.array(final_x), np.array(final_y1), np.array(final_y2)\n    final_x = np.transpose(final_x, [0, 2, 1])\n    return final_x, final_y1, final_y2","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:01.813741Z","iopub.execute_input":"2021-09-01T14:53:01.814093Z","iopub.status.idle":"2021-09-01T14:53:01.839169Z","shell.execute_reply.started":"2021-09-01T14:53:01.814058Z","shell.execute_reply":"2021-09-01T14:53:01.838291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, train_y1, train_y2 = pad(users_train)\ntest_x, test_y1, test_y2 = pad(users_test)\n#  Remove columns 5, 9, 10, 11, 12. 15, 16, 17, 18, ","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:01.846451Z","iopub.execute_input":"2021-09-01T14:53:01.848652Z","iopub.status.idle":"2021-09-01T14:53:08.121103Z","shell.execute_reply.started":"2021-09-01T14:53:01.848607Z","shell.execute_reply":"2021-09-01T14:53:08.120244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x.shape, test_x.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:08.122405Z","iopub.execute_input":"2021-09-01T14:53:08.122896Z","iopub.status.idle":"2021-09-01T14:53:08.128853Z","shell.execute_reply.started":"2021-09-01T14:53:08.122858Z","shell.execute_reply":"2021-09-01T14:53:08.127842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = train_x[..., :5]\nb = train_x[..., 6:9]\nc = train_x[..., 13:15]\ntrain_x_cat = np.concatenate((a, b, c), axis=-1)\ntrain_x_cat.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:08.13024Z","iopub.execute_input":"2021-09-01T14:53:08.130598Z","iopub.status.idle":"2021-09-01T14:53:08.225364Z","shell.execute_reply.started":"2021-09-01T14:53:08.130561Z","shell.execute_reply":"2021-09-01T14:53:08.224363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_cat=np.log(1+np.power(train_x_cat,0.25))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T15:40:31.642217Z","iopub.execute_input":"2021-09-01T15:40:31.642555Z","iopub.status.idle":"2021-09-01T15:40:32.538258Z","shell.execute_reply.started":"2021-09-01T15:40:31.642525Z","shell.execute_reply":"2021-09-01T15:40:32.537385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = test_x[..., :5]\nb = test_x[..., 6:9]\nc = test_x[..., 13:15]\ntest_x_cat = np.concatenate((a, b, c), axis=-1)\ntest_x_cat.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:08.226745Z","iopub.execute_input":"2021-09-01T14:53:08.227111Z","iopub.status.idle":"2021-09-01T14:53:08.252745Z","shell.execute_reply.started":"2021-09-01T14:53:08.227075Z","shell.execute_reply":"2021-09-01T14:53:08.251806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_cat=np.log(1+np.power(test_x_cat,0.25))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T15:40:49.137735Z","iopub.execute_input":"2021-09-01T15:40:49.138082Z","iopub.status.idle":"2021-09-01T15:40:49.362832Z","shell.execute_reply.started":"2021-09-01T15:40:49.13805Z","shell.execute_reply":"2021-09-01T15:40:49.361959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"serie_size =  train_x_cat.shape[1] # 30\nn_features =  train_x_cat.shape[2] # 10\n\nlstm_model_y2 = Sequential()\nlstm_model_y2.add(L.LSTM(10, input_shape=(serie_size, n_features), return_sequences=True))\nlstm_model_y2.add(L.LSTM(10, activation='relu', return_sequences=True))\nlstm_model_y2.add(L.LSTM(6, activation='relu'))\nlstm_model_y2.add(L.Dense(10, activation='relu'))\nlstm_model_y2.add(L.Dense(10, activation='relu'))\nlstm_model_y2.add(L.Dense(1))\nlstm_model_y2.summary()\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:08.254038Z","iopub.execute_input":"2021-09-01T14:53:08.254376Z","iopub.status.idle":"2021-09-01T14:53:10.481044Z","shell.execute_reply.started":"2021-09-01T14:53:08.25434Z","shell.execute_reply":"2021-09-01T14:53:10.480247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lr_scheduler(epoch, lr):\n    decay_rate = 0.5\n    decay_step = 95\n    if epoch % decay_step == 0 and epoch>0:\n        return lr * decay_rate\n    return lr\n\ncallbacks = [\n    keras.callbacks.LearningRateScheduler(lr_scheduler, verbose=1)\n]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:10.482356Z","iopub.execute_input":"2021-09-01T14:53:10.482693Z","iopub.status.idle":"2021-09-01T14:53:10.488097Z","shell.execute_reply.started":"2021-09-01T14:53:10.482657Z","shell.execute_reply":"2021-09-01T14:53:10.486927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 200\nbatch = 128*16*4*4\n\nlr = 0.01\n\nadam = optimizers.Adam(lr)\nlstm_model_y2.compile(loss='mse', optimizer=adam)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:10.489632Z","iopub.execute_input":"2021-09-01T14:53:10.490332Z","iopub.status.idle":"2021-09-01T14:53:10.512326Z","shell.execute_reply.started":"2021-09-01T14:53:10.490293Z","shell.execute_reply":"2021-09-01T14:53:10.511524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y2 = keras.models.load_model('../input/techniche-lstm-version-3-data/Y1_model.h5')\n# lstm_model_y2 = keras.models.load_model('./Y2_model.h5')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:53:10.515634Z","iopub.execute_input":"2021-09-01T14:53:10.515883Z","iopub.status.idle":"2021-09-01T14:53:10.939193Z","shell.execute_reply.started":"2021-09-01T14:53:10.515858Z","shell.execute_reply":"2021-09-01T14:53:10.938283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_history = lstm_model_y2.fit(train_x_cat, train_y2,\n                              validation_data=(test_x_cat, test_y2), \n                              batch_size=batch, \n                              epochs=300,\n                              verbose=1,\n                                callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T15:41:41.523592Z","iopub.execute_input":"2021-09-01T15:41:41.523911Z","iopub.status.idle":"2021-09-01T15:44:18.14594Z","shell.execute_reply.started":"2021-09-01T15:41:41.52388Z","shell.execute_reply":"2021-09-01T15:44:18.144926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intermediate_layer_model = Model(inputs=lstm_model_y2.input,\n                                 outputs=lstm_model_y2.layers[-2].output)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:55:59.539517Z","iopub.execute_input":"2021-09-01T14:55:59.539886Z","iopub.status.idle":"2021-09-01T14:55:59.548441Z","shell.execute_reply.started":"2021-09-01T14:55:59.539847Z","shell.execute_reply":"2021-09-01T14:55:59.547562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intermediate_layer_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:55:59.549839Z","iopub.execute_input":"2021-09-01T14:55:59.550281Z","iopub.status.idle":"2021-09-01T14:55:59.562825Z","shell.execute_reply.started":"2021-09-01T14:55:59.550246Z","shell.execute_reply":"2021-09-01T14:55:59.562047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb=intermediate_layer_model.predict(train_x_cat)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:55:59.564855Z","iopub.execute_input":"2021-09-01T14:55:59.565617Z","iopub.status.idle":"2021-09-01T14:56:20.466939Z","shell.execute_reply.started":"2021-09-01T14:55:59.565549Z","shell.execute_reply":"2021-09-01T14:56:20.466026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_emb=intermediate_layer_model.predict(test_x_cat)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:56:20.470346Z","iopub.execute_input":"2021-09-01T14:56:20.470612Z","iopub.status.idle":"2021-09-01T14:56:25.27719Z","shell.execute_reply.started":"2021-09-01T14:56:20.470585Z","shell.execute_reply":"2021-09-01T14:56:25.276275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_y1=CatBoostRegressor(depth=10, learning_rate=0.01)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T15:37:33.033975Z","iopub.execute_input":"2021-09-01T15:37:33.034349Z","iopub.status.idle":"2021-09-01T15:37:33.041027Z","shell.execute_reply.started":"2021-09-01T15:37:33.03431Z","shell.execute_reply":"2021-09-01T15:37:33.040222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_y1.fit(train_emb,train_y2,eval_set=(test_emb,test_y2))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T15:37:33.043085Z","iopub.execute_input":"2021-09-01T15:37:33.043722Z","iopub.status.idle":"2021-09-01T15:38:02.953218Z","shell.execute_reply.started":"2021-09-01T15:37:33.04368Z","shell.execute_reply":"2021-09-01T15:38:02.952419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y2.save('./Y2_model.h5')\n# model = load_model('my_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:56:46.230169Z","iopub.execute_input":"2021-09-01T14:56:46.230522Z","iopub.status.idle":"2021-09-01T14:56:46.269144Z","shell.execute_reply.started":"2021-09-01T14:56:46.230484Z","shell.execute_reply":"2021-09-01T14:56:46.26831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y2.predict(train_x_cat[:10]), train_y2[:10]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:56:46.270412Z","iopub.execute_input":"2021-09-01T14:56:46.27089Z","iopub.status.idle":"2021-09-01T14:56:46.806052Z","shell.execute_reply.started":"2021-09-01T14:56:46.270851Z","shell.execute_reply":"2021-09-01T14:56:46.80524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Y1","metadata":{}},{"cell_type":"code","source":"serie_size =  train_x_cat.shape[1] # 30\nn_features =  train_x_cat.shape[2] # 10\n\nlstm_model_y1 = Sequential()\nlstm_model_y1.add(L.LSTM(10, input_shape=(serie_size, n_features), return_sequences=True))\nlstm_model_y1.add(L.LSTM(10, input_shape=(serie_size, n_features), return_sequences=True))\nlstm_model_y1.add(L.LSTM(6, activation='relu'))\nlstm_model_y1.add(L.Dense(10, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y1.add(L.Dense(10, kernel_initializer='glorot_normal', activation='relu'))\nlstm_model_y1.add(L.Dense(1))\nlstm_model_y1.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:19:34.758537Z","iopub.execute_input":"2021-09-01T16:19:34.758872Z","iopub.status.idle":"2021-09-01T16:19:35.227993Z","shell.execute_reply.started":"2021-09-01T16:19:34.758833Z","shell.execute_reply":"2021-09-01T16:19:35.227041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 300\nbatch = 128*16*4*4\nlr = 0.01\n\nadam = optimizers.Adam(lr)\nlstm_model_y1.compile(loss='mse', optimizer=adam)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:19:36.248698Z","iopub.execute_input":"2021-09-01T16:19:36.249083Z","iopub.status.idle":"2021-09-01T16:19:36.264178Z","shell.execute_reply.started":"2021-09-01T16:19:36.249044Z","shell.execute_reply":"2021-09-01T16:19:36.263201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lstm_model_y1 = keras.models.load_model('../input/techniche-lstm-version-3-data/Y2_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:19:37.312927Z","iopub.execute_input":"2021-09-01T16:19:37.313284Z","iopub.status.idle":"2021-09-01T16:19:37.317293Z","shell.execute_reply.started":"2021-09-01T16:19:37.313253Z","shell.execute_reply":"2021-09-01T16:19:37.316254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.mkdir(\"Y1\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:55:04.776442Z","iopub.execute_input":"2021-09-01T16:55:04.776758Z","iopub.status.idle":"2021-09-01T16:55:04.781133Z","shell.execute_reply.started":"2021-09-01T16:55:04.776728Z","shell.execute_reply":"2021-09-01T16:55:04.780157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_checkpoint_callback = keras.callbacks.ModelCheckpoint(\n    filepath=\"./Y1\",\n    save_weights_only=True,\n    monitor='val_loss',\n    mode='min',\n    save_best_only=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:55:17.189541Z","iopub.execute_input":"2021-09-01T16:55:17.189847Z","iopub.status.idle":"2021-09-01T16:55:17.194152Z","shell.execute_reply.started":"2021-09-01T16:55:17.18982Z","shell.execute_reply":"2021-09-01T16:55:17.19314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lr_scheduler2(epoch, lr):\n    decay_rate = 0.5\n    decay_step = 125\n    if epoch % decay_step == 0 and epoch>0:\n        return lr * decay_rate\n    return lr\n\ncallbacks2 = [\n    keras.callbacks.LearningRateScheduler(lr_scheduler2, verbose=1),\n    model_checkpoint_callback\n]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:55:18.249276Z","iopub.execute_input":"2021-09-01T16:55:18.249753Z","iopub.status.idle":"2021-09-01T16:55:18.259321Z","shell.execute_reply.started":"2021-09-01T16:55:18.24971Z","shell.execute_reply":"2021-09-01T16:55:18.258442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_history = lstm_model_y1.fit(train_x_cat, train_y1,\n                              validation_data=(test_x_cat, test_y1), \n                              batch_size=batch, \n                              epochs=epochs,\n                              verbose=1,\n                                callbacks=callbacks2)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:55:21.387659Z","iopub.execute_input":"2021-09-01T16:55:21.388009Z","iopub.status.idle":"2021-09-01T16:57:34.2411Z","shell.execute_reply.started":"2021-09-01T16:55:21.387976Z","shell.execute_reply":"2021-09-01T16:57:34.240304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y1.load_weights('./Y1')\n# model = load_model('my_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:07:09.782732Z","iopub.execute_input":"2021-09-01T17:07:09.783067Z","iopub.status.idle":"2021-09-01T17:07:09.826361Z","shell.execute_reply.started":"2021-09-01T17:07:09.783017Z","shell.execute_reply":"2021-09-01T17:07:09.825609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intermediate_layer_model2 = Model(inputs=lstm_model_y1.input,\n                                 outputs=lstm_model_y1.layers[-2].output)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:26:15.731208Z","iopub.execute_input":"2021-09-01T16:26:15.731534Z","iopub.status.idle":"2021-09-01T16:26:15.74053Z","shell.execute_reply.started":"2021-09-01T16:26:15.731504Z","shell.execute_reply":"2021-09-01T16:26:15.739232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y1_emb_train=intermediate_layer_model2.predict(train_x_cat)\ny1_emb_test=intermediate_layer_model2.predict(test_x_cat)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:37:13.839351Z","iopub.execute_input":"2021-09-01T16:37:13.839684Z","iopub.status.idle":"2021-09-01T16:37:31.298734Z","shell.execute_reply.started":"2021-09-01T16:37:13.839654Z","shell.execute_reply":"2021-09-01T16:37:31.297854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_y1_=CatBoostRegressor(depth=6, learning_rate=0.05)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:42:02.928342Z","iopub.execute_input":"2021-09-01T16:42:02.928676Z","iopub.status.idle":"2021-09-01T16:42:02.93296Z","shell.execute_reply.started":"2021-09-01T16:42:02.928647Z","shell.execute_reply":"2021-09-01T16:42:02.932038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_y1_.fit(y1_emb_train,train_y1,eval_set=(y1_emb_test,test_y1))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:42:05.026268Z","iopub.execute_input":"2021-09-01T16:42:05.026577Z","iopub.status.idle":"2021-09-01T16:42:15.785374Z","shell.execute_reply.started":"2021-09-01T16:42:05.026549Z","shell.execute_reply":"2021-09-01T16:42:15.784586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lstm_model_y1.predict(train_x_cat[:2]), train_y1[:2]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:25:40.002953Z","iopub.execute_input":"2021-09-01T16:25:40.003292Z","iopub.status.idle":"2021-09-01T16:25:40.65761Z","shell.execute_reply.started":"2021-09-01T16:25:40.003263Z","shell.execute_reply":"2021-09-01T16:25:40.656726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"import json\nf = open('../input/techniche-lstm-version-3-data/x_test.json')\nX_test = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:47:38.745644Z","iopub.execute_input":"2021-09-01T16:47:38.745979Z","iopub.status.idle":"2021-09-01T16:47:43.975642Z","shell.execute_reply.started":"2021-09-01T16:47:38.745947Z","shell.execute_reply":"2021-09-01T16:47:43.97478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_pad(users, X=X_test, pad_size=30):\n    final_x = []\n    for user_id in tqdm(users):\n        x = np.array(X[str(user_id)]).T\n        pad_array = np.zeros((x.shape[0], pad_size - x.shape[1]))\n        x = np.concatenate((x, pad_array), axis=1)\n        final_x.append(x)\n    final_x = np.array(final_x)\n    final_x = np.transpose(final_x, [0, 2, 1])\n    return final_x","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:47:45.128789Z","iopub.execute_input":"2021-09-01T16:47:45.129224Z","iopub.status.idle":"2021-09-01T16:47:45.456401Z","shell.execute_reply.started":"2021-09-01T16:47:45.129185Z","shell.execute_reply":"2021-09-01T16:47:45.455433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/beyond-analysis/test.csv')\ntest_df.drop(columns = ['CATEGORY_1', 'CATEGORY_2'], axis=0, inplace = True)\n\nusers_sub = test_df['UNIQUE_IDENTIFIER'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:47:46.660702Z","iopub.execute_input":"2021-09-01T16:47:46.661033Z","iopub.status.idle":"2021-09-01T16:47:48.319782Z","shell.execute_reply.started":"2021-09-01T16:47:46.660987Z","shell.execute_reply":"2021-09-01T16:47:48.318932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_x = test_pad(users_sub)\n#  Remove columns 5, 9, 10, 11, 12. 15, 16, 17, 18, ","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:47:52.169763Z","iopub.execute_input":"2021-09-01T16:47:52.170115Z","iopub.status.idle":"2021-09-01T16:47:56.19409Z","shell.execute_reply.started":"2021-09-01T16:47:52.170083Z","shell.execute_reply":"2021-09-01T16:47:56.193212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_x.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:47:56.197972Z","iopub.execute_input":"2021-09-01T16:47:56.198245Z","iopub.status.idle":"2021-09-01T16:47:56.204573Z","shell.execute_reply.started":"2021-09-01T16:47:56.19822Z","shell.execute_reply":"2021-09-01T16:47:56.203428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = sub_x[..., :5]\nb = sub_x[..., 6:9]\nc = sub_x[..., 13:15]\nsub_x_cat = np.concatenate((a, b, c), axis=-1)\nsub_x_cat.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:47:56.733561Z","iopub.execute_input":"2021-09-01T16:47:56.733878Z","iopub.status.idle":"2021-09-01T16:47:56.797539Z","shell.execute_reply.started":"2021-09-01T16:47:56.73384Z","shell.execute_reply":"2021-09-01T16:47:56.79658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y1 = lstm_model_y1.predict(sub_x_cat)\nprint(pred_y1)\npred_y1.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:08:06.229578Z","iopub.execute_input":"2021-09-01T17:08:06.229923Z","iopub.status.idle":"2021-09-01T17:08:17.667274Z","shell.execute_reply.started":"2021-09-01T17:08:06.22989Z","shell.execute_reply":"2021-09-01T17:08:17.666357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y2_emb = intermediate_layer_model.predict(sub_x_cat)\n# print(pred_y2)\ny2_emb.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:08:50.844309Z","iopub.execute_input":"2021-09-01T17:08:50.84463Z","iopub.status.idle":"2021-09-01T17:09:07.986153Z","shell.execute_reply.started":"2021-09-01T17:08:50.844599Z","shell.execute_reply":"2021-09-01T17:09:07.985359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y2=cat_y1.predict(y2_emb)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:09:07.987511Z","iopub.execute_input":"2021-09-01T17:09:07.987845Z","iopub.status.idle":"2021-09-01T17:09:08.158137Z","shell.execute_reply.started":"2021-09-01T17:09:07.98781Z","shell.execute_reply":"2021-09-01T17:09:08.157274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y2.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:09:08.159845Z","iopub.execute_input":"2021-09-01T17:09:08.160201Z","iopub.status.idle":"2021-09-01T17:09:08.166505Z","shell.execute_reply.started":"2021-09-01T17:09:08.160165Z","shell.execute_reply":"2021-09-01T17:09:08.16558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=pd.read_csv(\"../input/cat-y1-pred/cat(1)(3).csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:09:08.167949Z","iopub.execute_input":"2021-09-01T17:09:08.168313Z","iopub.status.idle":"2021-09-01T17:09:08.20989Z","shell.execute_reply.started":"2021-09-01T17:09:08.168277Z","shell.execute_reply":"2021-09-01T17:09:08.209176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df[\"Y2\"]=pred_y2\nsub_df[\"Y1\"]=pred_y1","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:09:08.210946Z","iopub.execute_input":"2021-09-01T17:09:08.211286Z","iopub.status.idle":"2021-09-01T17:09:08.217676Z","shell.execute_reply.started":"2021-09-01T17:09:08.211248Z","shell.execute_reply":"2021-09-01T17:09:08.216578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"lstm-lstm-cat.csv\",index=None)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:09:08.219099Z","iopub.execute_input":"2021-09-01T17:09:08.219613Z","iopub.status.idle":"2021-09-01T17:09:08.506039Z","shell.execute_reply.started":"2021-09-01T17:09:08.219579Z","shell.execute_reply":"2021-09-01T17:09:08.505194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'UNIQUE_IDENTIFIER': users_sub.astype(int), \n                   'Y1': pred_y1[:, 0], 'Y2': pred_y2[:, 0]})\n\nsub.to_csv('sub.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T14:57:43.40738Z","iopub.execute_input":"2021-09-01T14:57:43.407624Z","iopub.status.idle":"2021-09-01T14:57:43.669919Z","shell.execute_reply.started":"2021-09-01T14:57:43.4076Z","shell.execute_reply":"2021-09-01T14:57:43.669006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}