{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:42:43.598354Z","iopub.execute_input":"2022-01-04T20:42:43.598647Z","iopub.status.idle":"2022-01-04T20:42:52.454951Z","shell.execute_reply.started":"2022-01-04T20:42:43.598615Z","shell.execute_reply":"2022-01-04T20:42:52.45413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set the needed data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport plotly.express as px\nimport glob\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset\n\nimport os\nimport time\n\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.metrics import mean_squared_error as mse\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import log_loss, accuracy_score\n\n\nimport py7zr\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        with py7zr.SevenZipFile(os.path.join(dirname, filename), mode='r') as z:\n            z.extractall()\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nos.environ['WANDB_CONSOLE'] = 'off'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-04T20:42:52.457098Z","iopub.execute_input":"2022-01-04T20:42:52.457375Z","iopub.status.idle":"2022-01-04T20:44:22.295499Z","shell.execute_reply.started":"2022-01-04T20:42:52.457338Z","shell.execute_reply":"2022-01-04T20:44:22.294734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3.a. Loading, Exploring and Preprocessing the Data\nWe have a few files that gives diffrent feature for the data. We will seperate them and move them into a comftarble one dataframe for validation test and train ","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv('./stores.csv')\nitems = pd.read_csv('./items.csv')\noil = pd.read_csv('./oil.csv', parse_dates=['date'])\n\ntrain = pd.read_csv('./train.csv',\n                   converters={'unit_sales': lambda x: np.log1p(float(x)) if float(x) > 0 else 0},\n                   parse_dates=['date'],\n                   skiprows= range(1,66458909) # seeing that \"train\" is around 5gb we \n                    # take about half of the data from 1.1.2016\n                   )\ntest = pd.read_csv('./test.csv', parse_dates=['date'])","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:44:22.29729Z","iopub.execute_input":"2022-01-04T20:44:22.297563Z","iopub.status.idle":"2022-01-04T20:47:51.401385Z","shell.execute_reply.started":"2022-01-04T20:44:22.297528Z","shell.execute_reply":"2022-01-04T20:47:51.400365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We want to split the data 20% and 80% but we also don't want to cut days. In our estimation 20% for validation fall in about 2017-15-2","metadata":{}},{"cell_type":"code","source":"cut = train.loc[train['date']>=pd.datetime(2017,1,1)]\nvalidation =cut.loc[cut['date']<=pd.datetime(2017,2,15)]\n\ntrain = train.loc[train['date']>pd.datetime(2017,2,15)] # we cut to only 2017 beacuse \n# the data set is huge","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:42:33.640399Z","iopub.execute_input":"2022-01-04T20:42:33.640984Z","iopub.status.idle":"2022-01-04T20:42:33.718608Z","shell.execute_reply.started":"2022-01-04T20:42:33.64087Z","shell.execute_reply":"2022-01-04T20:42:33.717379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will show the end of the of the validation and the start of train and test","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:02:22.906465Z","iopub.execute_input":"2021-12-30T20:02:22.906757Z","iopub.status.idle":"2021-12-30T20:02:22.927034Z","shell.execute_reply.started":"2021-12-30T20:02:22.906719Z","shell.execute_reply":"2021-12-30T20:02:22.926267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.tail()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:02:22.928223Z","iopub.execute_input":"2021-12-30T20:02:22.92848Z","iopub.status.idle":"2021-12-30T20:02:22.939155Z","shell.execute_reply.started":"2021-12-30T20:02:22.928444Z","shell.execute_reply":"2021-12-30T20:02:22.938266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:02:22.940695Z","iopub.execute_input":"2021-12-30T20:02:22.940982Z","iopub.status.idle":"2021-12-30T20:02:22.953854Z","shell.execute_reply.started":"2021-12-30T20:02:22.940947Z","shell.execute_reply":"2021-12-30T20:02:22.953087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now want to prepare our train, validation and test dataframe with all possible features (will be relvant later). Some features like item_nbr are a string so we want to encode them to integer numbers.","metadata":{}},{"cell_type":"code","source":"items['family'] = items['family'].astype('category').cat.codes\nstores['city'] = stores['city'].astype('category').cat.codes\nstores['state'] = stores['state'].astype('category').cat.codes\nstores['type'] = stores['type'].astype('category').cat.codes\nitems['item_enc'] = items['item_nbr'].astype('category').cat.codes\nitems['class'] = items['class'].astype('category').cat.codes\nstores['store_enc'] = stores['store_nbr'].astype('category').cat.codes\nstores['cluster'] = stores['cluster'].astype('category').cat.codes","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:01:04.057528Z","iopub.execute_input":"2022-01-04T19:01:04.057777Z","iopub.status.idle":"2022-01-04T19:01:04.079444Z","shell.execute_reply.started":"2022-01-04T19:01:04.057742Z","shell.execute_reply":"2022-01-04T19:01:04.078703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will merge the the values from items.csv\ndf_train = pd.merge(train,items,on='item_nbr')\ndf_val = pd.merge(validation,items, on='item_nbr' )\ndf_test = pd.merge(test,items, on='item_nbr')","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:01:04.080723Z","iopub.execute_input":"2022-01-04T19:01:04.081386Z","iopub.status.idle":"2022-01-04T19:01:15.117007Z","shell.execute_reply.started":"2022-01-04T19:01:04.081339Z","shell.execute_reply":"2022-01-04T19:01:15.116271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will merge the features based on the store number\ndf_train = pd.merge(df_train, stores, on='store_nbr')\ndf_val = pd.merge(df_val,stores, on='store_nbr' )\ndf_test = pd.merge(df_test,stores, on='store_nbr' )","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:01:15.118489Z","iopub.execute_input":"2022-01-04T19:01:15.118748Z","iopub.status.idle":"2022-01-04T19:01:23.343623Z","shell.execute_reply.started":"2022-01-04T19:01:15.118712Z","shell.execute_reply":"2022-01-04T19:01:23.342891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil = oil[oil['date']>= pd.datetime(2017,1,1)]\noil = oil.fillna(method='bfill')","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:01:23.345023Z","iopub.execute_input":"2022-01-04T19:01:23.345318Z","iopub.status.idle":"2022-01-04T19:01:23.354233Z","shell.execute_reply.started":"2022-01-04T19:01:23.345281Z","shell.execute_reply":"2022-01-04T19:01:23.353532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.merge(df_train, oil, on='date')\ndf_val = pd.merge(df_val,oil, on='date' )\ndf_test = pd.merge(df_test,oil, on='date' )","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:01:23.35551Z","iopub.execute_input":"2022-01-04T19:01:23.355941Z","iopub.status.idle":"2022-01-04T19:01:29.161201Z","shell.execute_reply.started":"2022-01-04T19:01:23.355902Z","shell.execute_reply":"2022-01-04T19:01:29.160446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['day'] = df_train['date'].dt.dayofweek\ndf_val['day'] = df_val['date'].dt.dayofweek\ndf_test['day'] = df_test['date'].dt.dayofweek","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:01:29.162432Z","iopub.execute_input":"2022-01-04T19:01:29.162689Z","iopub.status.idle":"2022-01-04T19:01:30.936508Z","shell.execute_reply.started":"2022-01-04T19:01:29.162656Z","shell.execute_reply":"2022-01-04T19:01:30.935725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2021-12-31T07:59:26.462946Z","iopub.execute_input":"2021-12-31T07:59:26.463632Z","iopub.status.idle":"2021-12-31T07:59:26.494196Z","shell.execute_reply.started":"2021-12-31T07:59:26.463594Z","shell.execute_reply":"2021-12-31T07:59:26.493538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will now define a dataset class for our forecasting","metadata":{}},{"cell_type":"markdown","source":"## 3.b. Solid Benchmark\n\nWe will try and do a solid benchmark using random forest algorithm","metadata":{}},{"cell_type":"code","source":"print('Starting classic ML')\nmaxrow = 300000\ny_train = df_train['unit_sales']\nx_train = df_train.drop(columns=['date','store_nbr','item_nbr','unit_sales'])\nrf = RandomForestRegressor(max_depth = x_train.shape[1], n_jobs = -1)\nbenchmark = rf.fit(x_train[:maxrow],y_train[:maxrow])\nprint('Finished fitting')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:02:49.030734Z","iopub.execute_input":"2021-12-30T20:02:49.031442Z","iopub.status.idle":"2021-12-30T20:03:34.534396Z","shell.execute_reply.started":"2021-12-30T20:02:49.031399Z","shell.execute_reply":"2021-12-30T20:03:34.533543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error as msle\nfrom sklearn.metrics import r2_score as r2\ny_val = df_val['unit_sales']\nx_val = df_val.drop(columns=['date','store_nbr','item_nbr','unit_sales'])\nmaxrow = 20000\npreds_bm = rf.predict(x_val[:maxrow])\nloss_val = msle(y_val[:maxrow].to_numpy(), preds_bm.astype(np.int))\nprint('The loss is: ',loss_val)\nacc = r2(y_val[:maxrow].to_numpy(), preds_bm)\nprint(f'The R^2 is {acc}')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:03:34.536138Z","iopub.execute_input":"2021-12-30T20:03:34.536415Z","iopub.status.idle":"2021-12-30T20:03:34.911505Z","shell.execute_reply.started":"2021-12-30T20:03:34.536368Z","shell.execute_reply":"2021-12-30T20:03:34.910355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.c. Preproccesing for embedding\n\nFirstly we will define a dataset that does not take date column. After that we will make a LSTM model with embedding layers, which in it we will create embbeding for the store_nbr and item_nbr ","metadata":{"execution":{"iopub.status.busy":"2021-12-28T18:59:48.117592Z","iopub.execute_input":"2021-12-28T18:59:48.117831Z","iopub.status.idle":"2021-12-28T18:59:48.125335Z","shell.execute_reply.started":"2021-12-28T18:59:48.117803Z","shell.execute_reply":"2021-12-28T18:59:48.124378Z"}}},{"cell_type":"code","source":"is_cuda = torch.cuda.is_available()\nif is_cuda:\n    device = torch.device(\"cuda\")\nelse:\n    device = torch.device(\"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:03:06.820448Z","iopub.execute_input":"2022-01-04T19:03:06.820707Z","iopub.status.idle":"2022-01-04T19:03:06.87295Z","shell.execute_reply.started":"2022-01-04T19:03:06.820676Z","shell.execute_reply":"2022-01-04T19:03:06.872034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimeDataset(Dataset):\n    def __init__(self, data):\n        # df to ndarray to tensor\n        \n        self.data = torch.from_numpy(data.iloc[:,1:].astype(np.float32).to_numpy()) # avoid unnecessary copy\n    \n    def __len__(self):\n        return len(self.data) \n\n    def __getitem__(self,idx):\n        # features\n        x = self.data[idx ,1:]\n        # labels\n        y = self.data[idx,0]\n        return x, y","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:02:50.67416Z","iopub.execute_input":"2022-01-04T19:02:50.674477Z","iopub.status.idle":"2022-01-04T19:02:50.681328Z","shell.execute_reply.started":"2022-01-04T19:02:50.674436Z","shell.execute_reply":"2022-01-04T19:02:50.679557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# counting the number of of values in item_nbr and store_nbr\ncount_store = stores['store_nbr'].nunique()\ncount_city = stores['city'].nunique()\ncount_item = items['item_nbr'].nunique()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:03:09.825975Z","iopub.execute_input":"2022-01-04T19:03:09.826561Z","iopub.status.idle":"2022-01-04T19:03:09.835897Z","shell.execute_reply.started":"2022-01-04T19:03:09.826522Z","shell.execute_reply":"2022-01-04T19:03:09.835187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SalesModel(nn.Module):\n    def __init__(self):\n        super(SalesModel,self).__init__()\n        \n        self.city_embd = nn.Embedding(count_city,5)\n        n_embd_city = self.city_embd.embedding_dim\n        self.str_embd = nn.Embedding(count_store,10)\n        n_embd_str = self.str_embd.embedding_dim\n        self.item_embd = nn.Embedding(count_item,100)\n        n_embd_item = self.item_embd.embedding_dim\n        self.day_embd = nn.Embedding(7,5)\n        n_embd_day = self.day_embd.embedding_dim\n        self.dropout = nn.Dropout(0.3)\n        n_emb = n_embd_city + n_embd_item + n_embd_str+n_embd_day\n        self.n_emb = n_emb\n        self.relu = nn.ReLU()\n        self.fc1 = nn.Linear(self.n_emb,70)\n        self.bn = nn.BatchNorm1d(70)\n        self.fc2 = nn.Linear(70,20)\n        self.fc3 = nn.Linear(20,1)\n        \n        \n    def forward(self,x):\n        #x = [e(x[:,i]) for i,e in enumerate(self.embeddings)]\n        a = self.item_embd(x[:,0].long())\n        b = self.str_embd(x[:,1].long())\n        c = self.city_embd(x[:,2].long())\n        d = self.day_embd(x[:,3].long())\n        x = torch.cat((a,b,c,d),1)\n        x = self.dropout(x)\n        #x = x.view(x.size(0),-1)\n        x = self.relu(self.fc1(x))\n        x = self.bn(x)\n        x = self.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\nmodel =SalesModel()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:03:12.091899Z","iopub.execute_input":"2022-01-04T19:03:12.092182Z","iopub.status.idle":"2022-01-04T19:03:12.146968Z","shell.execute_reply.started":"2022-01-04T19:03:12.092147Z","shell.execute_reply":"2022-01-04T19:03:12.146263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(val_loader,train_loader, learn_rate,s_model,EPOCHS=5):\n    \n    # Setting common hyperparameters\n    train_logs = []\n    val_logs = []\n    model = s_model.to(device)\n    \n    # Defining loss function and optimizer\n    criterion = nn.L1Loss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learn_rate)\n    \n    \n    print(\"Starting Training of GRU model\")\n    epoch_times = []\n    # Start training loop\n    for epoch in range(1,EPOCHS+1):\n        start_time = time.process_time()\n        model.train()\n        # intisalizing the weights\n        avg_loss = 0. # the . is for float as opposed to int\n        counter = 0\n        for x, label in train_loader:\n            counter += 1\n            model.zero_grad()\n            out = model(x.to(device).float())\n            \n            loss = criterion(out.float().squeeze(1), label.to(device).float())\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item()\n            if counter%4000 == 0:\n                print(\"Epoch {}......Step: {}/{}....... Average Loss for Epoch: {}\".format(epoch, counter, len(train_loader), avg_loss/counter))\n                train_logs.append(avg_loss/counter)\n        \n        print(\"Starting validation for epoch {}\".format(epoch))\n        \n        model.eval()\n        val_loss = 0. # the . is for float as opposed to int\n        counter_val = 0\n        val_preds = None\n        val_labels = None\n        \n        for x, label in val_loader:\n            \n            if val_labels is None:\n                val_labels = label.clone()\n            else:\n                val_labels = torch.cat((val_labels, label),dim=0)\n            \n            label = label.to(device, dtype=torch.int)\n            \n            counter_val += 1\n            \n            with torch.no_grad():\n                out = model(x.to(device).float())\n                loss = criterion(out.float().squeeze(1), label.to(device).float())\n            \n                val_loss += loss.item()\n                \n                preds = out.data.cpu()\n                \n                if val_preds is None:\n                    val_preds = preds\n                else:\n                    val_preds = torch.cat((val_preds,preds),dim=0)\n                    \n                if counter_val%2000 == 0:\n                    print(\"Epoch {}......Step: {}/{}....... Validation loss for Epoch: {}\".format(epoch, counter_val, len(val_loader), val_loss/counter_val))\n                    val_logs.append(val_loss / counter_val)\n        current_time = time.process_time()\n        print(\"Epoch {}/{} Done, Total Training Loss: {}\".format(epoch, EPOCHS, avg_loss/len(train_loader)))\n        print(\"Total Validation Loss: {}\".format(val_loss/len(val_loader)))\n        print(\"Time Elapsed for Epoch: {} seconds\".format(str(current_time-start_time)))\n        epoch_times.append(current_time-start_time)\n    print(\"Total Training Time: {} seconds\".format(str(sum(epoch_times))))\n    return model,train_logs,val_logs","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:03:35.079138Z","iopub.execute_input":"2021-12-30T20:03:35.079429Z","iopub.status.idle":"2021-12-30T20:03:35.104208Z","shell.execute_reply.started":"2021-12-30T20:03:35.07938Z","shell.execute_reply":"2021-12-30T20:03:35.103468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.d. Training the model and evaluating","metadata":{}},{"cell_type":"code","source":"batch_size = 1024\nnum_workers = 0\nfeatures = ['date','unit_sales','item_enc','store_enc','city','day']\nfeatures_test =['date','id','item_enc','store_enc','city','day']\n\ntest_set = TimeDataset(df_test.loc[:,features_test])\ntrain_set = TimeDataset(df_train.loc[:,features])\nval_set = TimeDataset(df_val.loc[:,features])\n\ntrain_loader = DataLoader(train_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True,drop_last=True)\nval_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True,drop_last=True)\ntest_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True,drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:03:35.10871Z","iopub.execute_input":"2021-12-30T20:03:35.109311Z","iopub.status.idle":"2021-12-30T20:03:38.834296Z","shell.execute_reply.started":"2021-12-30T20:03:35.109273Z","shell.execute_reply":"2021-12-30T20:03:38.833257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\nmodel_trained , train_logs, val_logs = train_model(val_loader,train_loader,learn_rate=lr,s_model=model)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:04:42.572196Z","iopub.execute_input":"2021-12-30T20:04:42.572777Z","iopub.status.idle":"2021-12-30T20:20:23.941209Z","shell.execute_reply.started":"2021-12-30T20:04:42.572737Z","shell.execute_reply":"2021-12-30T20:20:23.940312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Validation Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epochs')\nplt.plot(val_logs)\nplt.show()\nplt.title('Train Loss')\nplt.plot(train_logs)\nplt.ylabel('Loss')\nplt.xlabel('Epochs')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:20:23.943166Z","iopub.execute_input":"2021-12-30T20:20:23.943604Z","iopub.status.idle":"2021-12-30T20:20:24.356402Z","shell.execute_reply.started":"2021-12-30T20:20:23.943563Z","shell.execute_reply":"2021-12-30T20:20:24.35569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def NWRMSLE(y, pred, w):\n  y = y\n  pred = pred\n  w = w\n  \n  return (((w*(np.log(pred+1) - np.log(y+1)))**2).sum() / w.sum())**0.5","metadata":{"execution":{"iopub.status.busy":"2021-12-31T07:58:43.734886Z","iopub.execute_input":"2021-12-31T07:58:43.73543Z","iopub.status.idle":"2021-12-31T07:58:43.740139Z","shell.execute_reply.started":"2021-12-31T07:58:43.73539Z","shell.execute_reply":"2021-12-31T07:58:43.739344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, test_loader):\n    model.eval()\n    outputs = []\n    targets = []\n    start_time = time.process_time()\n    for x, label in test_loader:\n        inp = x\n        out = model(inp.to(device).long())\n        \n        outputs.append(out.cpu().detach().numpy().reshape(-1))\n    print(\"Evaluation Time: {}\".format(str(time.process_time()-start_time)))\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2021-12-31T08:21:11.194351Z","iopub.execute_input":"2021-12-31T08:21:11.194636Z","iopub.status.idle":"2021-12-31T08:21:11.201568Z","shell.execute_reply.started":"2021-12-31T08:21:11.194604Z","shell.execute_reply":"2021-12-31T08:21:11.200826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we have no labels in test set so we will just get them\nout = evaluate(model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:03:42.201611Z","iopub.status.idle":"2021-12-30T20:03:42.202102Z","shell.execute_reply.started":"2021-12-30T20:03:42.201855Z","shell.execute_reply":"2021-12-30T20:03:42.201893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In our run we got a train loss of about 3.84 and validation loss of about 5.32. We saw that in each epoch the loss in both the train and validation increased and in total the validation increased. We figure from this that we have overfitting to a  pretty small feature size.\n\n## 3.e. Better model\n\nWe will now add more features to our model to try and improve our prediction. Some of these features will not be embbeded.\n","metadata":{}},{"cell_type":"code","source":"count_fam = items['family'].nunique()\ncount_class = items['class'].nunique()\ncount_typ = stores['type'].nunique()\ncount_clst = stores['cluster'].nunique()\ncount_days = df_train['day'].nunique() # oddly enough there areonly 5 days not 7","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:03:21.839465Z","iopub.execute_input":"2022-01-04T19:03:21.839731Z","iopub.status.idle":"2022-01-04T19:03:21.915738Z","shell.execute_reply.started":"2022-01-04T19:03:21.839701Z","shell.execute_reply":"2022-01-04T19:03:21.915003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(count_clst,count_typ,count_class,count_fam)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:03:42.205411Z","iopub.status.idle":"2021-12-30T20:03:42.206278Z","shell.execute_reply.started":"2021-12-30T20:03:42.206028Z","shell.execute_reply":"2021-12-30T20:03:42.206053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class StoreNet(nn.Module):\n    def __init__(self):\n        super(StoreNet, self).__init__()\n        self.city_embd = nn.Embedding(count_city,10)\n        n_embd_city = self.city_embd.embedding_dim\n        self.str_embd = nn.Embedding(count_store,15)\n        n_embd_str = self.str_embd.embedding_dim\n        self.item_embd = nn.Embedding(count_item,100)\n        n_embd_item = self.item_embd.embedding_dim\n        self.day_embd = nn.Embedding(count_days,5)\n        n_embd_day = self.day_embd.embedding_dim\n        self.clst_embd = nn.Embedding(count_clst,5)\n        n_embd_clst = self.clst_embd.embedding_dim\n        self.class_embd = nn.Embedding(count_class,40)\n        n_embd_class = self.class_embd.embedding_dim\n        self.fam_embd = nn.Embedding(count_fam,20)\n        n_embd_fam = self.fam_embd.embedding_dim\n        \n        self.dropout1 = nn.Dropout(0.3)\n        self.dropout2 = nn.Dropout(0.2)\n        n_emb = n_embd_city + n_embd_item + n_embd_str+n_embd_day+n_embd_clst+n_embd_class+n_embd_fam\n        self.n_emb = n_emb\n        self.relu = nn.ReLU()\n        self.inpt = n_emb+2\n        \n        self.fc1 = nn.Linear(self.inpt,60)\n        self.bn_emb = nn.BatchNorm1d(self.n_emb)\n        self.bn1 = nn.BatchNorm1d(60)\n        self.bn2 = nn.BatchNorm1d(20)\n        self.fc2 = nn.Linear(60, 20)\n        self.fc3 = nn.Linear(20, 1)\n        self.relu = nn.ReLU()\n        \n    def forward(self, x):\n        a = self.item_embd(x[:,0].long())\n        b = self.str_embd(x[:,1].long())\n        c = self.city_embd(x[:,2].long())\n        d = self.day_embd(x[:,3].long())\n        e = self.clst_embd(x[:,4].long())\n        f = self.class_embd(x[:,5].long())\n        g = self.fam_embd(x[:,6].long())\n        seq = x[:,7:9]\n        \n        x = torch.cat((a,b,c,d,e,f,g),1)\n        \n        x = self.bn_emb(x)\n        x= self.dropout1(x)\n        x = torch.cat([x,seq],1)\n        x = x.view(x.size(0),-1)\n        \n        x = self.relu(self.fc1(x))\n        x = self.bn1(x)\n        x = self.dropout2(x)\n        \n        x = self.relu(self.fc2(x))\n        x = self.bn2(x)\n        \n        x = self.fc3(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:03:24.713661Z","iopub.execute_input":"2022-01-04T19:03:24.714228Z","iopub.status.idle":"2022-01-04T19:03:24.730412Z","shell.execute_reply.started":"2022-01-04T19:03:24.714184Z","shell.execute_reply":"2022-01-04T19:03:24.729724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 1024\nnum_workers = 2\nfeatures = ['date','unit_sales','item_enc','store_enc','city','day','cluster','class','family','dcoilwtico','onpromotion']\nfeatures_test =['date','id','item_enc','store_enc','city','day','cluster','class','family','dcoilwtico','onpromotion']\n\ntest_set = TimeDataset(df_test.loc[:,features_test])\ntrain_set = TimeDataset(df_train.loc[:,features])\nval_set = TimeDataset(df_val.loc[:,features])\n\ntrain_loader = DataLoader(train_set, batch_size=batch_size, shuffle=False, num_workers=num_workers,drop_last=True)#, pin_memory=True)\nval_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers,drop_last=True)#, pin_memory=True)\ntest_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers,drop_last=True)#, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:03:35.319572Z","iopub.execute_input":"2022-01-04T19:03:35.320036Z","iopub.status.idle":"2022-01-04T19:03:40.12124Z","shell.execute_reply.started":"2022-01-04T19:03:35.320002Z","shell.execute_reply":"2022-01-04T19:03:40.120427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\nmodel2, train_logs2, val_logs2 = train_model(val_loader,train_loader,learn_rate=lr,s_model=StoreNet())","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:20:29.557999Z","iopub.execute_input":"2021-12-30T20:20:29.558364Z","iopub.status.idle":"2021-12-30T20:37:16.15957Z","shell.execute_reply.started":"2021-12-30T20:20:29.558325Z","shell.execute_reply":"2021-12-30T20:37:16.158577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Validation Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epochs')\nplt.plot(val_logs2)\nplt.show()\nplt.title('Train Loss')\nplt.plot(train_logs2)\nplt.ylabel('Loss')\nplt.xlabel('Epochs')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T20:37:16.161203Z","iopub.execute_input":"2021-12-30T20:37:16.161479Z","iopub.status.idle":"2021-12-30T20:37:16.714961Z","shell.execute_reply.started":"2021-12-30T20:37:16.161439Z","shell.execute_reply":"2021-12-30T20:37:16.714287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now matter what we did with the model, adding features, increasing or decreasing the model complexity, playing with the learning rate, we still got roughly the same results as the first model with train loss of about 4.75 and unreasonable validation loss. We can also see that the validation loss increased as the model continue to train, which leads to us thinking we have over fitting ","metadata":{}},{"cell_type":"markdown","source":"### **When we run the code we got a warnning 'can only test child proccess' which accroding to the Pytorch forums is still an active problem that needs to be solved on their end**","metadata":{}},{"cell_type":"markdown","source":"## 3.f. Insights\n\nWe will plot the embedding of the store and item numbers we recived from the training","metadata":{}},{"cell_type":"code","source":"maxrow = 30000 # for mmemory allocation reasons\nfeatures_test =['item_enc','store_enc','city','day','cluster','class','family','dcoilwtico','onpromotion']\nx_test = torch.tensor(df_test.loc[:maxrow,features_test].astype(np.float32).values).to(device).float()\npreds = model2(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:01:29.173123Z","iopub.execute_input":"2022-01-04T20:01:29.173615Z","iopub.status.idle":"2022-01-04T20:01:29.191674Z","shell.execute_reply.started":"2022-01-04T20:01:29.173576Z","shell.execute_reply":"2022-01-04T20:01:29.190919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame()\nresult['id'] = df_test.loc[:maxrow,'id']\nresult['pred'] = preds.detach().cpu()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:29:33.215792Z","iopub.execute_input":"2022-01-04T20:29:33.216078Z","iopub.status.idle":"2022-01-04T20:29:33.229699Z","shell.execute_reply.started":"2022-01-04T20:29:33.216031Z","shell.execute_reply":"2022-01-04T20:29:33.228916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_redu =df_train.loc[:maxrow,:]\nindx = df_train_redu.loc[:,'item_enc'].unique()[:5]\ny_max = np.zeros(len(indx))\ny_min = np.zeros(len(indx))\nlist_result =[]\nfor i in range(0,len(indx)):\n    y_max[i] = np.amax(np.array(df_train_redu.loc[df_train_redu['item_enc']==indx[i],'unit_sales'],dtype=np.float))\n    y_min[i] = np.amin(np.array(df_train_redu.loc[df_train_redu['item_enc']==indx[i],'unit_sales'],dtype=np.float))\n    list_result += [result.loc[df_train_redu['item_enc']==indx[i],'pred']]","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:26:23.206231Z","iopub.execute_input":"2022-01-04T20:26:23.206612Z","iopub.status.idle":"2022-01-04T20:26:23.224044Z","shell.execute_reply.started":"2022-01-04T20:26:23.206563Z","shell.execute_reply":"2022-01-04T20:26:23.223363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(indx,y_max,color='r')\nplt.scatter(indx,y_min,color='r')\nfor i in range(0,len(indx)):\n    for j in list_result[i]:\n        plt.scatter(indx[i],j,color='b')\nplt.ylabel('Unit Sales')\nplt.xlabel('Item index')","metadata":{"execution":{"iopub.status.busy":"2022-01-04T20:28:09.448867Z","iopub.execute_input":"2022-01-04T20:28:09.449147Z","iopub.status.idle":"2022-01-04T20:28:10.210801Z","shell.execute_reply.started":"2022-01-04T20:28:09.449112Z","shell.execute_reply":"2022-01-04T20:28:10.210113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have plotted in red the max and min values of the unit sales of each of the items in the x axis, and in blue our prediction. We want to see our model prediceted in a reasonable range of values","metadata":{}},{"cell_type":"code","source":"# getting the embedding layer of store and item number\nstore_embd = model2.str_embd.weight\nitem_embd = model2.item_embd.weight","metadata":{"execution":{"iopub.status.busy":"2022-01-04T19:04:49.527088Z","iopub.execute_input":"2022-01-04T19:04:49.527751Z","iopub.status.idle":"2022-01-04T19:04:49.532452Z","shell.execute_reply.started":"2022-01-04T19:04:49.527712Z","shell.execute_reply":"2022-01-04T19:04:49.53174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import TSNE\n\nshow_str = {(i+1):v.detach().cpu().numpy() for i,v in enumerate(store_embd)}\ndf_show_str = pd.DataFrame.from_dict(show_str,orient='index')\ntsne = TSNE(perplexity=25,n_components=2,random_state=0)\nf = tsne.fit_transform(df_show_str) \nx = f[:,0]\ny = f[:,1]\nplt.figure(figsize=(12, 9)) \nplt.plot(x, y, 'ro')\n\nfor label, x, y in zip(df_show_str.index, x, y):\n        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points')\nplt.title('Store Embedding in 2 dimension')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T21:01:07.697324Z","iopub.execute_input":"2021-12-30T21:01:07.698024Z","iopub.status.idle":"2021-12-30T21:01:08.552728Z","shell.execute_reply.started":"2021-12-30T21:01:07.697979Z","shell.execute_reply":"2021-12-30T21:01:08.552035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_show = {(i+1):v.detach().cpu().numpy() for i,v in enumerate(item_embd)}\ndf_item_show = pd.DataFrame.from_dict(item_show,orient='index')\nf2 = tsne.fit_transform(df_item_show) \n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T21:01:16.565684Z","iopub.execute_input":"2021-12-30T21:01:16.565986Z","iopub.status.idle":"2021-12-30T21:01:54.411782Z","shell.execute_reply.started":"2021-12-30T21:01:16.565951Z","shell.execute_reply":"2021-12-30T21:01:54.410872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x2 = f2[:,0]\ny2 = f2[:,1]\nplt.figure(figsize=(12, 9)) \nplt.plot(x2, y2, 'ro')\nplt.title('Item Embedding in 2 dimension no text')\nplt.show()\n\nplt.figure(figsize=(12, 9)) \nplt.plot(x2, y2, 'ro')\nfor label, x, y in zip(df_item_show.index, x2, y2):\n        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points')\nplt.title('Item Embedding in 2 dimension with text')        ","metadata":{"execution":{"iopub.status.busy":"2021-12-30T21:01:54.416052Z","iopub.execute_input":"2021-12-30T21:01:54.417908Z","iopub.status.idle":"2021-12-30T21:02:20.426327Z","shell.execute_reply.started":"2021-12-30T21:01:54.416425Z","shell.execute_reply":"2021-12-30T21:02:20.42572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that store are pretty much uniformly distribute, but the items all algin on some curve, which we can't infer much from","metadata":{}},{"cell_type":"markdown","source":"## 3.g. Feature Extractor\n\nWe will now use the model that we got as a feature extractor and use classical machine learning to get a similar result. We will agian use random forest ","metadata":{}},{"cell_type":"code","source":"class StoreNet_fex(nn.Module):\n    def __init__(self):\n        super(StoreNet_fex, self).__init__()\n        self.backbone = model2\n        \n    def forward(self, x):\n        a = self.backbone.item_embd(x[:,0].long())\n        b = self.backbone.str_embd(x[:,1].long())\n        c = self.backbone.city_embd(x[:,2].long())\n        d = self.backbone.day_embd(x[:,3].long())\n        e = self.backbone.clst_embd(x[:,4].long())\n        f = self.backbone.class_embd(x[:,5].long())\n        g = self.backbone.fam_embd(x[:,6].long())\n        seq = x[:,7:9]\n        \n        x = torch.cat((a,b,c,d,e,f,g),1)\n        \n        x = self.backbone.bn_emb(x)\n        x= self.backbone.dropout1(x)\n        x = torch.cat([x,seq],1)\n        x = x.view(x.size(0),-1)\n        \n        x = self.backbone.relu(self.backbone.fc1(x))\n        x = self.backbone.bn1(x)\n        x = self.backbone.dropout2(x)\n        \n        x = self.backbone.relu(self.backbone.fc2(x))\n        x = self.backbone.bn2(x)\n    \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-12-31T07:41:03.923724Z","iopub.execute_input":"2021-12-31T07:41:03.924019Z","iopub.status.idle":"2021-12-31T07:41:03.934109Z","shell.execute_reply.started":"2021-12-31T07:41:03.923965Z","shell.execute_reply":"2021-12-31T07:41:03.933235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maxrow = 300000\nweight = df_val.loc[:maxrow,'perishable']\nfor z in weight:\n    if weight[z]==0:\n        weight[z]=1\n    else:\n        weight[z]=1.25","metadata":{"execution":{"iopub.status.busy":"2021-12-31T08:13:13.572644Z","iopub.execute_input":"2021-12-31T08:13:13.573463Z","iopub.status.idle":"2021-12-31T08:13:18.131121Z","shell.execute_reply.started":"2021-12-31T08:13:13.573423Z","shell.execute_reply":"2021-12-31T08:13:18.130342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error as msle\nmaxrow = 300000\nfeatures = ['item_enc','store_enc','city','day','cluster','class','family','dcoilwtico','onpromotion']\nprint('Getting model as a feature extrator')\nmodel_fex = StoreNet_fex()\ny_train = torch.tensor(df_train.loc[:maxrow,'unit_sales'].astype(np.float32).values).to(device).float()\nx_train = torch.tensor(df_train.loc[:maxrow,features].astype(np.float32).values).to(device).float()\nx_new = model_fex(x_train)\n\nprint('Starting classic ML')\n\nrf = RandomForestRegressor(max_depth = x_train.shape[1], n_jobs = -1)\nfittedmodel = rf.fit(x_new.detach().cpu(),y_train.cpu())\nprint('Finished fitting')","metadata":{"execution":{"iopub.status.busy":"2021-12-31T07:41:37.95691Z","iopub.execute_input":"2021-12-31T07:41:37.957618Z","iopub.status.idle":"2021-12-31T07:44:37.52402Z","shell.execute_reply.started":"2021-12-31T07:41:37.957577Z","shell.execute_reply":"2021-12-31T07:44:37.523237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['item_enc','store_enc','city','day','cluster','class','family','dcoilwtico','onpromotion']\ny_val = torch.tensor(df_val.loc[:maxrow,'unit_sales'].astype(np.float32).values).to(device).float()\nx_val = torch.tensor(df_val.loc[:maxrow,features].astype(np.float32).values).to(device).float()\nx_val_new = model_fex(x_val)\n\npreds_bm = rf.predict(x_val_new.detach().cpu())\nloss_val = NWRMSLE(y_val.cpu().numpy(), preds_bm.astype(np.int),weight)\nprint('The loss is: ',loss_val)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T08:15:23.177057Z","iopub.execute_input":"2021-12-31T08:15:23.177734Z","iopub.status.idle":"2021-12-31T08:15:23.984217Z","shell.execute_reply.started":"2021-12-31T08:15:23.177696Z","shell.execute_reply":"2021-12-31T08:15:23.983373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we have no labels in test set so we will just get them\nout = evaluate(model2, test_loader)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T08:21:15.739826Z","iopub.execute_input":"2021-12-31T08:21:15.741238Z","iopub.status.idle":"2021-12-31T08:21:43.984298Z","shell.execute_reply.started":"2021-12-31T08:21:15.741191Z","shell.execute_reply":"2021-12-31T08:21:43.9834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that with a feature extractor we get a bad loss of about 2.54 with only a small part of the rows. We still do not fully understand where our neural network failed to predict the correct values, but using is as a feature extractor has led to even the classical machine learning which gave good result at the start of this question, to fail.","metadata":{}}]}