{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Make use of text in the item names.  Ths notebook does not adjust for seasonality.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\n\nPATH_CSV = '/kaggle/input/applications-of-deep-learning-wustlfall-2022/beach_demand_forecast/'\n\ndf_sales = pd.read_csv(os.path.join(PATH_CSV,\"sales_train.csv\"))\ndf_items = pd.read_csv(os.path.join(PATH_CSV,\"items.csv\"))\ndf_resturant = pd.read_csv(os.path.join(PATH_CSV,\"resturants.csv\"))\n\ndf_sales.date = pd.to_datetime(df_sales.date, errors='coerce') ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-30T00:24:37.835697Z","iopub.execute_input":"2022-10-30T00:24:37.836154Z","iopub.status.idle":"2022-10-30T00:24:38.055311Z","shell.execute_reply.started":"2022-10-30T00:24:37.836083Z","shell.execute_reply":"2022-10-30T00:24:38.053824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load the Glove Embeddings","metadata":{}},{"cell_type":"code","source":"!wget -c \"https://nlp.stanford.edu/data/glove.6B.zip\"\n!unzip glove.6B.zip","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:24:38.057078Z","iopub.execute_input":"2022-10-30T00:24:38.057327Z","iopub.status.idle":"2022-10-30T00:27:45.618302Z","shell.execute_reply.started":"2022-10-30T00:24:38.057303Z","shell.execute_reply":"2022-10-30T00:27:45.617107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.test.utils import datapath, get_tmpfile\nfrom gensim.models import KeyedVectors\nfrom gensim.scripts.glove2word2vec import glove2word2vec\n\nglove_file = 'glove.6B.300d.txt'\ntmp_file = get_tmpfile(\"test_word2vec.txt\")\n_ = glove2word2vec(glove_file, tmp_file)\nw2vec_model = KeyedVectors.load_word2vec_format(tmp_file)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:27:45.619596Z","iopub.execute_input":"2022-10-30T00:27:45.619863Z","iopub.status.idle":"2022-10-30T00:30:44.794165Z","shell.execute_reply.started":"2022-10-30T00:27:45.619818Z","shell.execute_reply":"2022-10-30T00:30:44.792020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NLP Demo","metadata":{}},{"cell_type":"code","source":"w2vec_model.most_similar(positive=['woman', 'king'], negative=['man'])","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:44.808857Z","iopub.execute_input":"2022-10-30T00:30:44.810475Z","iopub.status.idle":"2022-10-30T00:30:45.012291Z","shell.execute_reply.started":"2022-10-30T00:30:44.810409Z","shell.execute_reply":"2022-10-30T00:30:45.011394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w2vec_model['dog'].shape","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.013643Z","iopub.execute_input":"2022-10-30T00:30:45.014180Z","iopub.status.idle":"2022-10-30T00:30:45.020203Z","shell.execute_reply.started":"2022-10-30T00:30:45.014144Z","shell.execute_reply":"2022-10-30T00:30:45.019408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Perform NLP Prediction","metadata":{}},{"cell_type":"code","source":"def process_title(model, name):\n  v = None\n  i = 0\n  for word in name.split(' '):\n    word = word.lower()\n    if word == 'vegi': word = \"vegetable\"\n    if word == 'smoothy': word = \"malt\"\n    i+=1\n    if v is None and word in model:\n      v=model[word].copy()\n    elif word in model:\n      v+=model[word]\n  v/=i\n  return v\n\nitem_lookup = {}\nfor i, name in zip(list(df_items.id),list(df_items.name)):\n  v = process_title(w2vec_model,name)\n  item_lookup[i] = v\n\n#r = process_title(w2vec_model, 'breaded fish with vegetables meal')\n#print(r)\nprint(len(item_lookup))","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.021676Z","iopub.execute_input":"2022-10-30T00:30:45.022318Z","iopub.status.idle":"2022-10-30T00:30:45.036616Z","shell.execute_reply.started":"2022-10-30T00:30:45.022274Z","shell.execute_reply":"2022-10-30T00:30:45.035712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Utility function to create sequences.","metadata":{}},{"cell_type":"code","source":"def series_to_supervised(data, window=1, lag=1, dropnan=True):\n    cols, names = list(), list()\n    # Input sequence (t-n, ... t-1)\n    for i in range(window, 0, -1):\n        cols.append(data.shift(i))\n        names += [('%s(t-%d)' % (col, i)) for col in data.columns]\n    # Current timestep (t=0)\n    cols.append(data)\n    names += [('%s(t)' % (col)) for col in data.columns]\n    # Target timestep (t=lag)\n    cols.append(data.shift(-lag))\n    names += [('%s(t+%d)' % (col, lag)) for col in data.columns]\n    # Put it all together\n    agg = pd.concat(cols, axis=1)\n    agg.columns = names\n    # Drop rows with NaN values\n    if dropnan:\n        agg.dropna(inplace=True)\n    return agg","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.038106Z","iopub.execute_input":"2022-10-30T00:30:45.038667Z","iopub.status.idle":"2022-10-30T00:30:45.046504Z","shell.execute_reply.started":"2022-10-30T00:30:45.038633Z","shell.execute_reply":"2022-10-30T00:30:45.045753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Join the items and sales tables so that we can look up the store id for each item.","metadata":{}},{"cell_type":"code","source":"df_items2 = df_items[['id','store_id']]\ndf_train = df_sales.merge(df_items2,left_on='item_id',right_on='id')\ndf_train[['date','item_id','item_count','store_id']]\n\ndf_train = df_train.sort_values('date').groupby(['item_id', 'store_id', 'date'], as_index=False)\ndf_train = df_train.agg({'item_count':['mean']})\ndf_train.columns = ['item', 'store', 'date', 'sales']\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.047764Z","iopub.execute_input":"2022-10-30T00:30:45.048294Z","iopub.status.idle":"2022-10-30T00:30:45.156184Z","shell.execute_reply.started":"2022-10-30T00:30:45.048260Z","shell.execute_reply":"2022-10-30T00:30:45.155285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['dow'] = df_train['date'].dt.dayofweek\ndf_train['doy'] = df_train['date'].dt.dayofyear\n\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.157416Z","iopub.execute_input":"2022-10-30T00:30:45.157925Z","iopub.status.idle":"2022-10-30T00:30:45.185116Z","shell.execute_reply.started":"2022-10-30T00:30:45.157893Z","shell.execute_reply":"2022-10-30T00:30:45.184029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Build the sequence data.","metadata":{}},{"cell_type":"code","source":"window = 29\nfuture_span = 30\nseries = series_to_supervised(df_train.drop('date', axis=1), window=window, lag=future_span)\nseries.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.187618Z","iopub.execute_input":"2022-10-30T00:30:45.187900Z","iopub.status.idle":"2022-10-30T00:30:45.706249Z","shell.execute_reply.started":"2022-10-30T00:30:45.187872Z","shell.execute_reply":"2022-10-30T00:30:45.705371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Remove sequences that did not have enough data.","metadata":{}},{"cell_type":"code","source":"\n# Remove edge cases, where there were not enough values to complete a series\nlast_item = 'item(t-%d)' % window\nlast_store = 'store(t-%d)' % window\nlast_dow = 'dow(t-%d)' % window\nlast_doy = 'doy(t-%d)' % window\nseries = series[(series['store(t)'] == series[last_store])]\nseries = series[(series['item(t)'] == series[last_item])]\n#series = series[(series['dow(t)'] == series[last_dow])]\n#series = series[(series['doy(t)'] == series[last_doy])]","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.707342Z","iopub.execute_input":"2022-10-30T00:30:45.707571Z","iopub.status.idle":"2022-10-30T00:30:45.824787Z","shell.execute_reply.started":"2022-10-30T00:30:45.707548Z","shell.execute_reply":"2022-10-30T00:30:45.823799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will predict with sales, and our engineered features.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom keras.utils.np_utils import to_categorical   \n\ndef drop_column(df, col):\n  columns_to_drop = [('%s(t+%d)' % (col, future_span))]\n  for i in range(window, 0, -1):\n      columns_to_drop += [('%s(t-%d)' % (col, i))]\n  df.drop(columns_to_drop, axis=1, inplace=True, errors='ignore')\n  df.drop([f\"{col}(t)\"], axis=1, inplace=True, errors='ignore')\n\ndef cat_seq(df, col):\n  return to_categorical(df[col].values)\n\n# Label\nlabels_col = 'sales(t+%d)' % future_span\nlabels = series[labels_col]\nseries.drop(labels_col, axis=1, inplace=True)\nseries.drop('item(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('store(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('dow(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('doy(t+%d)' % future_span, axis=1, inplace=True)\n\n# Get sales sequences\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"dow\")\ndrop_column(series2, \"doy\")\nsales_series = series2.values\n\n# Day of week as a number\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"doy\")\ndrop_column(series2, \"sales\")\ndow_series = series2.values\n\n# Get day of year sequences\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"dow\")\ndrop_column(series2, \"sales\")\ndoy_series = series2.values\n\n# Day of year\nt1 = sales_series.reshape(sales_series.shape + (1,))\nt2 = dow_series.reshape(dow_series.shape + (1,)) \nt3 = doy_series.reshape(doy_series.shape + (1,))\n\n# Create predictors (x)\nvec_size = w2vec_model['test'].shape[0]\n\nlst = []\nfor item in list(series['item(t-1)']):\n  lst.append(item_lookup[item])\n\nx1 = np.concatenate([t1,t2,t3],axis=2)\nx2 = np.concatenate(lst).reshape((series.shape[0],vec_size))\n\nx = [x1,x2]","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:45.826050Z","iopub.execute_input":"2022-10-30T00:30:45.826359Z","iopub.status.idle":"2022-10-30T00:30:52.980293Z","shell.execute_reply.started":"2022-10-30T00:30:45.826327Z","shell.execute_reply":"2022-10-30T00:30:52.978993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(t1.shape)\nprint(t2.shape)\nprint(t3.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:52.981656Z","iopub.execute_input":"2022-10-30T00:30:52.982369Z","iopub.status.idle":"2022-10-30T00:30:52.988611Z","shell.execute_reply.started":"2022-10-30T00:30:52.982339Z","shell.execute_reply":"2022-10-30T00:30:52.987501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract the predictors (x sequences) and the label (future prediction)","metadata":{}},{"cell_type":"code","source":"TEST_SIZE = 0.4\n\nmask = np.random.random(size=x[0].shape[0])<TEST_SIZE\n\nX_train = []\nX_valid = []\n\nfor subx in x:\n  X_train.append(subx[~mask])\n  X_valid.append(subx[mask])\n\nY_train = labels.values[~mask]\nY_valid = labels.values[mask]\n\nprint('Train set shape x1:', X_train[0].shape)\nprint('Train set shape x2:', X_train[1].shape)\nprint('Validation set shape x1:', X_valid[0].shape)\nprint('Validation set shape x2:', X_valid[1].shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:52.989884Z","iopub.execute_input":"2022-10-30T00:30:52.990146Z","iopub.status.idle":"2022-10-30T00:30:53.119497Z","shell.execute_reply.started":"2022-10-30T00:30:52.990121Z","shell.execute_reply":"2022-10-30T00:30:53.117618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Construct the neural network.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf \nfrom keras.models import Sequential, Model\nfrom keras.layers.convolutional import Conv1D, MaxPooling1D\nfrom keras.layers import Dense, LSTM, RepeatVector, TimeDistributed, Flatten, Dropout, concatenate, Input\nimport keras\n\nepochs = 500\nbatch = 256\nlr = 0.0003\nadam = tf.keras.optimizers.Adam(lr)\n\nmodel = Sequential()\n\nA1 = Input(shape=(X_train[0].shape[1], X_train[0].shape[2]),name='A1')\nA2 = Conv1D(filters=64, kernel_size=8, activation='relu')(A1)\nA3 = MaxPooling1D(pool_size=2)(A2)\nA4 = Flatten()(A3)\nA5 = Dense(50, activation='relu')(A4)\nA6 = Dropout(0.2)(A5)\n\nB1 = Input(shape=X_train[1].shape[1],name='B1')\nB2 = Dense(16, activation='relu',name='B2')(B1)\n\nM1 = concatenate([A6,B2])\nM2 = Dense(1,name='M2')(M1)\n\nmodel = Model(inputs=[A1, B1],outputs=[M2])\nmodel.compile(loss='mse', optimizer=adam)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:53.121081Z","iopub.execute_input":"2022-10-30T00:30:53.121444Z","iopub.status.idle":"2022-10-30T00:30:53.688387Z","shell.execute_reply.started":"2022-10-30T00:30:53.121409Z","shell.execute_reply":"2022-10-30T00:30:53.687424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fit the neural network.","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nmonitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5, \n        verbose=1, mode='auto', restore_best_weights=True)\n\ncnn_history = model.fit(X_train, Y_train, callbacks=[monitor],\n    validation_data=(X_valid, Y_valid), epochs=epochs, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:30:53.689796Z","iopub.execute_input":"2022-10-30T00:30:53.690525Z","iopub.status.idle":"2022-10-30T00:33:33.392054Z","shell.execute_reply.started":"2022-10-30T00:30:53.690455Z","shell.execute_reply":"2022-10-30T00:33:33.390346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict and evaluate the validation data.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nimport numpy as np\n\ncnn_train_pred = model.predict(X_train)\ncnn_valid_pred = model.predict(X_valid)\nprint('Train rmse:', np.sqrt(mean_squared_error(Y_train, cnn_train_pred)))\nprint('Validation rmse:', np.sqrt(mean_squared_error(Y_valid, cnn_valid_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:33:33.393699Z","iopub.execute_input":"2022-10-30T00:33:33.394735Z","iopub.status.idle":"2022-10-30T00:33:39.332336Z","shell.execute_reply.started":"2022-10-30T00:33:33.394633Z","shell.execute_reply":"2022-10-30T00:33:39.331005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot the training curve.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfig = plt.figure()\nplt.plot(cnn_history.history['loss'], label='Train loss')\nplt.plot(cnn_history.history['val_loss'], label='Validation loss')\nfig.legend()\nfig.suptitle('CNN')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"MSE\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:33:39.334017Z","iopub.execute_input":"2022-10-30T00:33:39.334626Z","iopub.status.idle":"2022-10-30T00:33:39.520688Z","shell.execute_reply.started":"2022-10-30T00:33:39.334595Z","shell.execute_reply":"2022-10-30T00:33:39.519917Z"},"trusted":true},"execution_count":null,"outputs":[]}]}