{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook makes use of NLP and images, but does not adjust for seasonality.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\n\nPATH_CSV = '/kaggle/input/applications-of-deep-learning-wustlfall-2022/beach_demand_forecast/'\nPATH_CAM = '/kaggle/input/applications-of-deep-learning-wustlfall-2022/beach_demand_forecast/cam/'\n\ndf_sales = pd.read_csv(os.path.join(PATH_CSV,\"sales_train.csv\"))\ndf_items = pd.read_csv(os.path.join(PATH_CSV,\"items.csv\"))\ndf_resturant = pd.read_csv(os.path.join(PATH_CSV,\"resturants.csv\"))\n\ndf_sales.date = pd.to_datetime(df_sales.date, errors='coerce') ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-30T01:48:07.477097Z","iopub.execute_input":"2022-10-30T01:48:07.477624Z","iopub.status.idle":"2022-10-30T01:48:07.636461Z","shell.execute_reply.started":"2022-10-30T01:48:07.477585Z","shell.execute_reply":"2022-10-30T01:48:07.635116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install YOLOv5","metadata":{}},{"cell_type":"code","source":"import sys\n\n!git clone https://github.com/ultralytics/yolov5 --tag 6.2  # clone\n!mv /kaggle/working/6.2 /kaggle/working/yolov5\n%pip install -qr /kaggle/working/yolov5/requirements.txt  # install\nsys.path.insert(0,'/kaggle/working/yolov5/')\n\nimport torch\nimport utils\ndisplay = utils.notebook_init()  # checks","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:21:28.307329Z","iopub.execute_input":"2022-10-30T01:21:28.307778Z","iopub.status.idle":"2022-10-30T01:21:50.012185Z","shell.execute_reply.started":"2022-10-30T01:21:28.307744Z","shell.execute_reply":"2022-10-30T01:21:50.010692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Demonstrate YOLOv5","metadata":{}},{"cell_type":"code","source":"!rm -R /kaggle/working/yolov5/runs/detect/*\n!mkdir /kaggle/working/images\n!cp /kaggle/input/applications-of-deep-learning-wustlfall-2022/beach_demand_forecast/cam/2020_01_01.jpg /kaggle/working/images","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:46:40.495068Z","iopub.execute_input":"2022-10-30T01:46:40.496265Z","iopub.status.idle":"2022-10-30T01:46:43.797276Z","shell.execute_reply.started":"2022-10-30T01:46:40.496224Z","shell.execute_reply":"2022-10-30T01:46:43.795793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import Image\n\n!python /kaggle/working/yolov5/detect.py --weights yolov5s.pt --img 1024 \\\n  --conf 0.25 --source /kaggle/working/images/\n\nURL = '/kaggle/working/yolov5/runs/detect/exp/2020_01_01.jpg'\nImage(filename=URL, width=300)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:46:48.042573Z","iopub.execute_input":"2022-10-30T01:46:48.043105Z","iopub.status.idle":"2022-10-30T01:46:54.377754Z","shell.execute_reply.started":"2022-10-30T01:46:48.043062Z","shell.execute_reply":"2022-10-30T01:46:54.376445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\n# Model\nyolo_model = torch.hub.load('ultralytics/yolov5', 'yolov5s')  # or yolov5n - yolov5x6, custom\n\n# Images\nimg = os.path.join(PATH_CAM, '2020_01_01.jpg')\n\n# Inference\nresults = yolo_model(img)\n\n# Results\ndf = results.pandas().xyxy[0]\ndf","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:48:12.956960Z","iopub.execute_input":"2022-10-30T01:48:12.957393Z","iopub.status.idle":"2022-10-30T01:48:13.995171Z","shell.execute_reply.started":"2022-10-30T01:48:12.957357Z","shell.execute_reply":"2022-10-30T01:48:13.993585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Data from Street Images","metadata":{}},{"cell_type":"code","source":"import os\nfrom os import walk\nimport datetime\nimport tqdm\nimport pandas as pd\n\nfilenames = next(walk(PATH_CAM), (None, None, []))[2]  \n\nlist_date = []\nlist_people = []\n\nfor file in tqdm.tqdm(filenames):\n  if file=='1.jpg': continue\n  filename = os.path.join(PATH_CAM, file)\n  results = yolo_model(filename)\n  df = results.pandas().xyxy[0]\n  people = len(df[df.name=='person'])\n  dt = datetime.datetime.strptime(file[:10], '%Y_%m_%d')\n  list_date.append(dt)\n  list_people.append(people)\n\ndf_street_view = pd.DataFrame({'date':list_date,'people':list_people})\ndf_street_view","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:53:02.394453Z","iopub.execute_input":"2022-10-30T01:53:02.394897Z","iopub.status.idle":"2022-10-30T01:56:08.699428Z","shell.execute_reply.started":"2022-10-30T01:53:02.394863Z","shell.execute_reply":"2022-10-30T01:56:08.698268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Network Code from NLP Notebook","metadata":{}},{"cell_type":"code","source":"def process_title(model, name):\n  v = None\n  i = 0\n  for word in name.split(' '):\n    word = word.lower()\n    if word == 'vegi': word = \"vegetable\"\n    if word == 'smoothy': word = \"malt\"\n    i+=1\n    if v is None and word in model:\n      v=model[word].copy()\n    elif word in model:\n      v+=model[word]\n  v/=i\n  return v\n\ndef series_to_supervised(data, window=1, lag=1, dropnan=True):\n    cols, names = list(), list()\n    # Input sequence (t-n, ... t-1)\n    for i in range(window, 0, -1):\n        cols.append(data.shift(i))\n        names += [('%s(t-%d)' % (col, i)) for col in data.columns]\n    # Current timestep (t=0)\n    cols.append(data)\n    names += [('%s(t)' % (col)) for col in data.columns]\n    # Target timestep (t=lag)\n    cols.append(data.shift(-lag))\n    names += [('%s(t+%d)' % (col, lag)) for col in data.columns]\n    # Put it all together\n    agg = pd.concat(cols, axis=1)\n    agg.columns = names\n    # Drop rows with NaN values\n    if dropnan:\n        agg.dropna(inplace=True)\n    return agg\n    \ndef drop_column(df, col):\n  columns_to_drop = [('%s(t+%d)' % (col, future_span))]\n  for i in range(window, 0, -1):\n      columns_to_drop += [('%s(t-%d)' % (col, i))]\n  df.drop(columns_to_drop, axis=1, inplace=True, errors='ignore')\n  df.drop([f\"{col}(t)\"], axis=1, inplace=True, errors='ignore')\n\ndef cat_seq(df, col):\n  return to_categorical(df[col].values)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:09:24.028395Z","iopub.execute_input":"2022-10-30T02:09:24.028908Z","iopub.status.idle":"2022-10-30T02:09:24.045504Z","shell.execute_reply.started":"2022-10-30T02:09:24.028873Z","shell.execute_reply":"2022-10-30T02:09:24.044486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the Glove Embeddings","metadata":{}},{"cell_type":"code","source":"!wget -c \"https://nlp.stanford.edu/data/glove.6B.zip\"\n!unzip glove.6B.zip\n\nfrom gensim.test.utils import datapath, get_tmpfile\nfrom gensim.models import KeyedVectors\nfrom gensim.scripts.glove2word2vec import glove2word2vec\n\nglove_file = 'glove.6B.300d.txt'\ntmp_file = get_tmpfile(\"test_word2vec.txt\")\n_ = glove2word2vec(glove_file, tmp_file)\nw2vec_model = KeyedVectors.load_word2vec_format(tmp_file)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:58:43.436325Z","iopub.execute_input":"2022-10-30T01:58:43.436745Z","iopub.status.idle":"2022-10-30T02:05:12.030374Z","shell.execute_reply.started":"2022-10-30T01:58:43.436710Z","shell.execute_reply":"2022-10-30T02:05:12.028601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engineer Time Series Features:","metadata":{}},{"cell_type":"code","source":"item_lookup = {}\nfor i, name in zip(list(df_items.id),list(df_items.name)):\n  v = process_title(w2vec_model,name)\n  item_lookup[i] = v\n\n# Join the items and sales tables so that we can look up the store id for each item.\ndf_items2 = df_items[['id','store_id']]\ndf_train = df_sales.merge(df_items2,left_on='item_id',right_on='id')\ndf_train[['date','item_id','item_count','store_id']]\n\n# Merge people counts (new)\ntemp = len(df_train)\ndf_train = df_train.merge(df_street_view)\nassert len(df_train) == temp\n\n# Sort/agg\ndf_train = df_train.sort_values('date').groupby(['item_id', 'store_id', 'date'], as_index=False)\ndf_train = df_train.agg({'item_count':['mean'],'people':['mean']})\ndf_train.columns = ['item', 'store', 'date', 'sales','people']\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:09:31.232263Z","iopub.execute_input":"2022-10-30T02:09:31.232812Z","iopub.status.idle":"2022-10-30T02:09:31.431881Z","shell.execute_reply.started":"2022-10-30T02:09:31.232769Z","shell.execute_reply":"2022-10-30T02:09:31.430657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['dow'] = df_train['date'].dt.dayofweek\ndf_train['doy'] = df_train['date'].dt.dayofyear\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:09:36.539824Z","iopub.execute_input":"2022-10-30T02:09:36.541489Z","iopub.status.idle":"2022-10-30T02:09:36.590868Z","shell.execute_reply.started":"2022-10-30T02:09:36.541436Z","shell.execute_reply":"2022-10-30T02:09:36.589368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Build the sequence data.","metadata":{}},{"cell_type":"code","source":"window = 29\nfuture_span = 30\nseries = series_to_supervised(df_train.drop('date', axis=1), window=window, lag=future_span)\n\n# Remove edge cases, where there were not enough values to complete a series\nlast_item = 'item(t-%d)' % window\nlast_store = 'store(t-%d)' % window\nlast_dow = 'dow(t-%d)' % window\nlast_doy = 'doy(t-%d)' % window\nseries = series[(series['store(t)'] == series[last_store])]\nseries = series[(series['item(t)'] == series[last_item])]","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:09:40.218136Z","iopub.execute_input":"2022-10-30T02:09:40.219225Z","iopub.status.idle":"2022-10-30T02:09:41.368575Z","shell.execute_reply.started":"2022-10-30T02:09:40.219165Z","shell.execute_reply":"2022-10-30T02:09:41.366990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will predict with sales, and our engineered features.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom keras.utils.np_utils import to_categorical   \n\n# Label\nlabels_col = 'sales(t+%d)' % future_span\nlabels = series[labels_col]\nseries.drop(labels_col, axis=1, inplace=True)\nseries.drop('item(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('store(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('dow(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('doy(t+%d)' % future_span, axis=1, inplace=True)\nseries.drop('people(t+%d)' % future_span, axis=1, inplace=True)\n\n# Get sales sequences\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"dow\")\ndrop_column(series2, \"doy\")\ndrop_column(series2, \"people\")\nsales_series = series2.values\n\n# Day of week as a number\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"doy\")\ndrop_column(series2, \"sales\")\ndrop_column(series2, \"people\")\ndow_series = series2.values\n\n# Get day of year sequences\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"dow\")\ndrop_column(series2, \"sales\")\ndrop_column(series2, \"people\")\ndoy_series = series2.values\n\n# Get number of people sequences\nseries2 = series.copy()\ndrop_column(series2, \"item\")\ndrop_column(series2, \"store\")\ndrop_column(series2, \"dow\")\ndrop_column(series2, \"sales\")\ndrop_column(series2, \"doy\")\npeople_series = series2.values\n\n\n# Create x\nt1 = sales_series.reshape(sales_series.shape + (1,))\nt2 = dow_series.reshape(dow_series.shape + (1,)) \nt3 = doy_series.reshape(doy_series.shape + (1,))\nt4 = people_series.reshape(people_series.shape + (1,))\nx1 = np.concatenate([t1,t2,t3,t4],axis=2)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:09:46.320156Z","iopub.execute_input":"2022-10-30T02:09:46.320650Z","iopub.status.idle":"2022-10-30T02:09:54.280951Z","shell.execute_reply.started":"2022-10-30T02:09:46.320613Z","shell.execute_reply":"2022-10-30T02:09:54.279660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(t1.shape)\nprint(t2.shape)\nprint(t3.shape)\nprint(t4.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:10:04.070115Z","iopub.execute_input":"2022-10-30T02:10:04.071182Z","iopub.status.idle":"2022-10-30T02:10:04.077426Z","shell.execute_reply.started":"2022-10-30T02:10:04.071112Z","shell.execute_reply":"2022-10-30T02:10:04.076553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create predictors (x)\nvec_size = w2vec_model['test'].shape[0]\n\nlst = []\nfor item in list(series['item(t-1)']):\n  lst.append(item_lookup[item])\n\nx2 = np.concatenate(lst).reshape((series.shape[0],vec_size))\n\nx = [x1,x2]","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:10:07.388436Z","iopub.execute_input":"2022-10-30T02:10:07.388903Z","iopub.status.idle":"2022-10-30T02:10:07.526455Z","shell.execute_reply.started":"2022-10-30T02:10:07.388869Z","shell.execute_reply":"2022-10-30T02:10:07.525054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the Network\n\nExtract the predictors (x sequences) and the label (future prediction)","metadata":{}},{"cell_type":"code","source":"TEST_SIZE = 0.4\n\nmask = np.random.random(size=x[0].shape[0])<TEST_SIZE\n\nX_train = []\nX_valid = []\n\nfor subx in x:\n  X_train.append(subx[~mask])\n  X_valid.append(subx[mask])\n\nY_train = labels.values[~mask]\nY_valid = labels.values[mask]\n\nprint('Train set shape x1:', X_train[0].shape)\nprint('Train set shape x2:', X_train[1].shape)\nprint('Validation set shape x1:', X_valid[0].shape)\nprint('Validation set shape x2:', X_valid[1].shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:10:10.951651Z","iopub.execute_input":"2022-10-30T02:10:10.952123Z","iopub.status.idle":"2022-10-30T02:10:11.144731Z","shell.execute_reply.started":"2022-10-30T02:10:10.952087Z","shell.execute_reply":"2022-10-30T02:10:11.142871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Construct the neural network.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf \nfrom keras.models import Sequential, Model\nfrom keras.layers.convolutional import Conv1D, MaxPooling1D\nfrom keras.layers import Dense, LSTM, RepeatVector, TimeDistributed, Flatten, Dropout, concatenate, Input\nimport keras\n\nepochs = 500\nbatch = 256\nlr = 0.0003\nadam = tf.keras.optimizers.Adam(lr)\n\nmodel = Sequential()\n\nA1 = Input(shape=(X_train[0].shape[1], X_train[0].shape[2]),name='A1')\nA2 = Conv1D(filters=64, kernel_size=8, activation='relu')(A1)\nA3 = MaxPooling1D(pool_size=2)(A2)\nA4 = Flatten()(A3)\nA5 = Dense(50, activation='relu')(A4)\nA6 = Dropout(0.2)(A5)\n\nB1 = Input(shape=X_train[1].shape[1],name='B1')\nB2 = Dense(16, activation='relu',name='B2')(B1)\n\nM1 = concatenate([A6,B2])\nM2 = Dense(1,name='M2')(M1)\n\nmodel = Model(inputs=[A1, B1],outputs=[M2])\nmodel.compile(loss='mse', optimizer=adam)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:10:16.736966Z","iopub.execute_input":"2022-10-30T02:10:16.737915Z","iopub.status.idle":"2022-10-30T02:10:17.567018Z","shell.execute_reply.started":"2022-10-30T02:10:16.737867Z","shell.execute_reply":"2022-10-30T02:10:17.565595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fit the neural network.","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nmonitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5, \n        verbose=1, mode='auto', restore_best_weights=True)\n\ncnn_history = model.fit(X_train, Y_train, callbacks=[monitor],\n    validation_data=(X_valid, Y_valid), epochs=epochs, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T02:10:22.084928Z","iopub.execute_input":"2022-10-30T02:10:22.085346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict and evaluate the validation data.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nimport numpy as np\n\ncnn_train_pred = model.predict(X_train)\ncnn_valid_pred = model.predict(X_valid)\nprint('Train rmse:', np.sqrt(mean_squared_error(Y_train, cnn_train_pred)))\nprint('Validation rmse:', np.sqrt(mean_squared_error(Y_valid, cnn_valid_pred)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot the training curve.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfig = plt.figure()\nplt.plot(cnn_history.history['loss'], label='Train loss')\nplt.plot(cnn_history.history['val_loss'], label='Validation loss')\nfig.legend()\nfig.suptitle('CNN')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"MSE\")\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}