{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install py7zr\n!pip install shap\n!pip install catboost","metadata":{"id":"kVDutuuKrWGC","outputId":"5b0b7e4f-94b8-489b-9029-0eeee7ba7e6a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\nimport os\nimport py7zr\nimport pandas as pd\nimport pickle\nimport datetime\nfrom datetime import date, timedelta\nimport matplotlib.pyplot as plt\nimport shap\nimport numpy as np\nimport tensorflow as tf\nfrom catboost import CatBoostRegressor\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import precision_score, \\\n    recall_score, confusion_matrix, classification_report, \\\n    accuracy_score, f1_score, mean_squared_error\nfrom tensorflow.keras.layers import Input, Embedding, dot, Dot, add,Flatten, concatenate,Dropout, Dense,BatchNormalization\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import plot_model \nfrom sklearn.preprocessing import MinMaxScaler","metadata":{"id":"wOEAOOSXrO26","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_DIR = '/kaggle/input/favoritagrocerysalesforecastingextracted/'","metadata":{"id":"njZUpOgNshvL","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Extractor","metadata":{"id":"R8YSVs5MWv_4"}},{"cell_type":"code","source":"\noil = pd.read_csv(PATH_DIR+'oil.csv', parse_dates=[\"date\"])\nitems = pd.read_csv(PATH_DIR+'items.csv')\nstores = pd.read_csv(PATH_DIR+'stores.csv')\n\ntest = pd.read_csv(PATH_DIR+'test.csv', parse_dates=[\"date\"])\n\ntrain = pd.read_csv(\n    PATH_DIR+'train.csv', usecols=[1, 2, 3, 4, 5],\n    dtype={'onpromotion': bool},\n    converters={'unit_sales': lambda u: np.log1p(\n        float(u)) if float(u) > 0 else 0},\n    parse_dates=[\"date\"],\n    skiprows=range(1, 66458909)  # 2016-01-01\n)\ntrain.unit_sales = train.unit_sales.astype(pd.np.float64)\n\ntrain_1 = train.loc[train.date>=pd.datetime(2017,5,1)]\ntrain = train_1.loc[train_1.date>pd.datetime(2017,5,25)]\nvalidation = train_1.loc[train_1.date<=pd.datetime(2017,5,25)]","metadata":{"id":"Q_NfWNevrkFH","outputId":"90fec579-65cd-4bcc-d12a-d6966e186c1e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"id":"EZi-Fw8Lx8k2","outputId":"e2559773-e765-4824-a2b1-3fc1c0f18a30","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.head()","metadata":{"id":"8cN2rUytGx63","outputId":"95a9ee26-9c64-4124-addb-4190cd91bd0c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Length of the train',len(train))\nprint('Length of the validation',len(validation))","metadata":{"id":"EnkLKTgUG4ri","outputId":"4e82219a-42df-48d0-c056-64b2295cb2b8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"id":"M8tGo_WMDo6s","outputId":"6870b22c-092a-4fed-ccfe-26fbf18b21a5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are exploring the csv files that we have and print the information about them","metadata":{}},{"cell_type":"code","source":"dataframes = [(oil,\"oil\"),(items,\"items\"),(stores,\"stores\"),(test,\"test\"),(train,\"train\")]\nfor df in dataframes:\n  print(\"df NAME:\", df[1])\n  print(df[0].head())\n  print(df[0].apply(lambda x: sum(x.isnull()),axis=0))\n  print('========================')\n","metadata":{"id":"MYOhioQJt79h","outputId":"11b46ef5-a7ce-4648-f331-7fd7946d51da","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Filling nan values in oil table with 10 moving average technique","metadata":{"id":"29BIowonzF8-"}},{"cell_type":"code","source":"dates = train.date\ndates = dates.append(test.date)\ndates = dates.append(validation.date)\nunique_dates = pd.DataFrame({'date': dates.unique()})\n\noil = oil.merge(unique_dates, on='date', how='outer')\noil = oil.sort_values('date')\n\noil = oil.fillna(oil['dcoilwtico'].rolling(10, min_periods=1, center=True, win_type='gaussian')\n                 .mean(std=1).to_frame())","metadata":{"id":"YynHI0Gfx29q","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Preprossing the datas, transforam valutes into [0...length] format and preformes minMax scaling\n  on сontinuous variables","metadata":{}},{"cell_type":"code","source":"def preprocessing(items,stores,oil,train,test,validation):\n  \"\"\"\n  Preprossing the datas, transforam valutes into [0...length] format and preformes minMax scaling\n  on сontinuous variables\n\n  \"\"\"\n  item_enc = {v:k for (k,v) in enumerate(items.item_nbr.unique())}\n  class_enc = {v:k for (k,v) in enumerate(items['class'].unique())}\n  family_enc = {v:k for (k,v) in enumerate(items['family'].unique())}\n\n  store_enc = {v:k for (k,v) in enumerate(stores.store_nbr.unique())}\n  city_enc = {v:k for (k,v) in enumerate(stores['city'].unique())}\n  type_enc = {v:k for (k,v) in enumerate(stores['type'].unique())}\n  state_enc = {v:k for (k,v) in enumerate(stores['state'].unique())}\n  cluster_enc = {v:k for (k,v) in enumerate(stores['cluster'].unique())}\n\n  train['store_nbr'] = [store_enc[x] for x in train['store_nbr']]\n  train['item_nbr'] = [item_enc[x] for x in train['item_nbr']]\n\n  test['store_nbr'] = [store_enc[x] for x in test['store_nbr']]\n  test['item_nbr'] = [item_enc[x] for x in test['item_nbr']]\n\n  validation['store_nbr'] = [store_enc[x] for x in validation['store_nbr']]\n  validation['item_nbr'] = [item_enc[x] for x in validation['item_nbr']]\n\n  items['item_nbr'] = [item_enc[x] for x in items['item_nbr']]\n  items['class'] = [class_enc[x] for x in items['class']]\n  items['family'] = [family_enc[x] for x in items['family']]\n\n  stores['store_nbr'] = [store_enc[x] for x in stores['store_nbr']]\n  stores['city'] = [city_enc[x] for x in stores['city']]\n  stores['type'] = [type_enc[x] for x in stores['type']]\n  stores['state'] = [state_enc[x] for x in stores['state']]\n  stores['cluster'] = [cluster_enc[x] for x in stores['cluster']]\n\n  scaler = MinMaxScaler()\n\n  scaler.fit(oil[['dcoilwtico']])\n  oil['dcoilwtico'] = scaler.transform(oil[['dcoilwtico']])\n  oil['dcoilwtico'] = oil['dcoilwtico'].astype('float32')\n\n  return items, stores, oil, train, test, validation, item_enc, cluster_enc, store_enc\n\n\nitems, stores, oil, train, test, validation, item_enc, cluster_enc, store_enc = preprocessing(items,stores,oil,train,test,validation)","metadata":{"id":"epb982GIQypg","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mergging different dataframes into one dataframe and describe the informaion","metadata":{}},{"cell_type":"code","source":"def merge_dfs(main_df, oil, stores, items):\n    \"\"\"\n    Merging different dataframes into one df\n    Args:\n     main_df(DataFrame)\n     oil(DataFrame)\n     stores(DataFrame)\n     items(DataFrame)\n    Output:\n      df. merged data frame\n      wights. wights of the samples for evalution\n    \"\"\"\n    df = main_df.merge(oil, on='date', how='left')\n    df = df.merge(stores, on='store_nbr', how='left')\n    df = df.merge(items, on='item_nbr', how='left')\n\n    df['day'] = df.date.dt.dayofweek\n\n\n\n    wightes = df['perishable'].map({0:1.0, 1:1.25})\n    df = df.drop(['date','perishable'], axis=1)\n\n    return df, wightes\n\n\ndef information(items,stores):\n \"\"\"\n Returns the legnth of unique vaues of the relavent features\n \"\"\"\n n_items = len(items.item_nbr.unique())\n\n n_class = len(items['class'].unique())\n\n n_family = len(items['family'].unique())\n\n n_store = len(stores['store_nbr'].unique())\n\n n_type = len(stores['type'].unique())\n\n n_city = len(stores['city'].unique())\n\n n_state = len(stores['state'].unique())\n\n n_cluster = len(stores['cluster'].unique())\n\n return n_class, n_store, n_type, n_city, n_state, n_cluster, n_items, n_family\n\n\ndf_train, wightes_train = merge_dfs(train, oil, stores, items)\ndf_test, wightes_test= merge_dfs(test, oil, stores, items)\ndf_validation, wightes_validtion = merge_dfs(validation, oil, stores, items)\n\nn_class, n_store, n_type, n_city, n_state, n_cluster, n_items, n_family = information(items,stores)\n\nprint(df_train.describe())\nprint()\nprint(df_train.dtypes)\n","metadata":{"id":"Xan6dcm7-qpf","outputId":"ae9d3ac9-f3b8-43c9-f03a-5f68633cc0fc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"id":"_n2Kdc0Eht33","outputId":"22a3a675-b6e2-4f72-ff5c-bbfb1bd4c6e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def simple_to_Xy_array(df):\n  \"\"\"\n  Extract features for the first DP regressor model\n  Args:\n    df(DataFrame):  data\n  Output:\n    x. features\n    y. target\n  \"\"\"\n  features = ['store_nbr',\n              'item_nbr',\n              'cluster',\n              'day']\n\n  target_class = 'unit_sales'\n\n  X = []\n  y = []\n  for f in features:\n      X.append(df[f])\n  if target_class is not None:    \n      y = df[target_class]\n  return X, y\n\ndef to_Xy_array(df):\n  \"\"\"\n  Extract features for the second DP regressor model\n    Args:\n    df(DataFrame):  data\n  Output:\n    x. features\n    y. target\n  \"\"\"\n  features = ['store_nbr',\n              'item_nbr',\n              'cluster',\n              'day',\n              'city',\n              'family',\n              'type',\n              'state',\n              'class',\n              'onpromotion',\n              'dcoilwtico']\n\n  target_class = 'unit_sales'\n\n  X = []\n  y = []\n  for f in features:\n      X.append(df[f])\n  if target_class is not None:    \n      y = df[target_class]\n  return X, y\n\ndef to_Xy_df(df):\n  \"\"\"\n  Extract features for ML model\n    Args:\n    df(DataFrame):  data\n  Output:\n    x. features\n    y. target\n  \"\"\"\n  features = ['store_nbr',\n              'item_nbr',\n              'onpromotion',\n              'dcoilwtico',\n              'cluster',\n              'family',\n              'city',\n              'day']\n  target_class = 'unit_sales'\n\n  X = df.drop(columns=[target_class])\n  y = df[target_class]\n\n  return X, y\n        \nx_train_df, Y_train_df = to_Xy_df(df_train)\nx_train_s, Y_train_s = simple_to_Xy_array(df_train)\nx_train, Y_train = to_Xy_array(df_train)\n\nx_val_df, Y_val_df = to_Xy_df(df_validation)\nx_val_s, Y_val_s = simple_to_Xy_array(df_validation)\nx_val, Y_val = to_Xy_array(df_validation)\n\n","metadata":{"id":"qjqZJ2oKEXpR","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def NWRMSLE(y, pred, w):\n  \"\"\"\n  Normalized wighted root mean squer logirtimic error metric\n  Args:\n    y(np.array):  actuall label\n    pred(np.arry):  prediction label\n    w(np.array):  wights of the samples\n  Output:\n    int. metric score\n  \"\"\"\n  y = np.array(y)\n  pred = np.array(pred)\n  w = np.array(w)\n  \n  return (((w*(np.log(pred+1) - np.log(y+1)))**2).sum() / w.sum())**0.5","metadata":{"id":"3SPBqsQgmGhc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3b. training CatBoost Model","metadata":{"id":"qZbY0_cKZxNT"}},{"cell_type":"code","source":"model = CatBoostRegressor()\nhistory_cat = model.fit(x_train_df, Y_train_df)\n","metadata":{"id":"o33vsNqgLxm3","outputId":"19dbc7f5-5000-4e5e-99bf-73e2d62de92d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(x_val_df)\npredictions = np.clip(predictions, 0, max(0, predictions.max()))\nscore = NWRMSLE(Y_val_df, predictions, wightes_validtion.values)\nprint('NWRMSLE CAT BOOST:',score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def shap_expliner(model,X_train):\n  shap.initjs()\n  explainer = shap.Explainer(model)\n  shap_values = explainer(X_train)\n  shap.plots.bar(shap_values)\n  # visualize the first prediction's explanation\n#   shap.plots.waterfall(shap_values[0])\n  # summarize the effects of all the features\n#   shap.plots.beeswarm(shap_values)\n\n\nshap_expliner(history_cat,x_train_df)","metadata":{"id":"heO7GjfCgrXk","outputId":"fafae16c-617c-4a72-d905-158a8446da19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# #3c+e. Preprocessing steps to create an embedding models\n\n* For the first model we embedded the following features:\n\n  1.   store id\n  2.  item id\n  3. clusted\n  4. day\n\n* For the second model we also embedded city, family, type, state, class, features and added onpromotion ,dcoilwtico features to the model.\n\n","metadata":{"id":"7BrT1P4gbxZ9"}},{"cell_type":"code","source":"msle = tf.keras.losses.MeanSquaredLogarithmicError()\n\ndef get_embedding_model():\n  \"\"\"\n  First simple model for 3c\n  \"\"\"\n  store_in = Input(shape=(1,), dtype='int64', name='dim_store')\n  s = Embedding(n_store, 15, input_length=1, embeddings_regularizer=l2(1e-4),)(store_in)\n\n  item_in = Input(shape=(1,), dtype='int64', name='dim_item')\n  m = Embedding(n_items, 50, input_length=1, embeddings_regularizer=l2(1e-4))(item_in)\n\n  cluster_in = Input(shape=(1,), dtype='int64', name='din_cluster')\n  c = Embedding(n_cluster, 5, input_length=1, embeddings_regularizer=l2(1e-4))(cluster_in)\n\n  day_in = Input(shape=(1,), dtype='int64', name='day_in')\n  d = Embedding(7, 4, input_length=1, embeddings_regularizer=l2(1e-4))(day_in)\n\n\n\n  x = concatenate([s, m, c, d])\n  x = Flatten()(x)\n  x = BatchNormalization()(x)\n  x = Dense(100, activation='relu')(x)\n  x = BatchNormalization()(x)\n  x = Dropout(0.5)(x)\n  x = Dense(50, activation='relu')(x)\n  x = Dropout(0.5)(x)\n  x = Dense(1)(x)\n  nn = Model([store_in,item_in,cluster_in,day_in], x)\n  nn.compile(Adam(0.001), loss='mse',metrics=['mae',msle])\n  nn.summary()\n  return nn\n\ndef get_embedding_model_2():\n  \"\"\"\n  More complex model for 3e\n  \"\"\"\n  store_in = Input(shape=(1,), dtype='int64', name='dim_store')\n  s = Embedding(n_store, 15, input_length=1, embeddings_regularizer=l2(1e-4),)(store_in)\n\n  item_in = Input(shape=(1,), dtype='int64', name='dim_item')\n  m = Embedding(n_items, 50, input_length=1, embeddings_regularizer=l2(1e-4))(item_in)\n\n  cluster_in = Input(shape=(1,), dtype='int64', name='din_cluster')\n  c = Embedding(n_cluster, 5, input_length=1, embeddings_regularizer=l2(1e-4))(cluster_in)\n\n  day_in = Input(shape=(1,), dtype='int64', name='day_in')\n  d = Embedding(7, 4, input_length=1, embeddings_regularizer=l2(1e-4))(day_in)\n\n  city_in = Input(shape=(1,), dtype='int64', name='city_in')\n  t = Embedding(n_city, 7, input_length=1, embeddings_regularizer=l2(1e-4))(city_in)\n\n  family_in = Input(shape=(1,), dtype='int64', name='family_in')\n  f = Embedding(n_family, 15, input_length=1, embeddings_regularizer=l2(1e-4))(family_in)\n\n  type_in = Input(shape=(1,), dtype='int64', name='type_in')\n  t = Embedding(n_type, 3, input_length=1, embeddings_regularizer=l2(1e-4))(type_in)\n  \n  state_in = Input(shape=(1,), dtype='int64', name='state_in')\n  st = Embedding(n_state, 5, input_length=1, embeddings_regularizer=l2(1e-4))(state_in)\n\n  class_in = Input(shape=(1,), dtype='int64', name='class_inday_in')\n  cl = Embedding(n_class, 20, input_length=1, embeddings_regularizer=l2(1e-4))(class_in)\n\n  onpromotion_in = Input(shape=(1,1), dtype='float32', name='onpromotion_in')\n  dcoilwtico_in = Input(shape=(1,1), dtype='float32', name='dcoilwtico_in')\n\n\n  x = concatenate([s,m,c,d,t,f,t,st,cl,onpromotion_in,dcoilwtico_in])\n  x = Flatten()(x)\n  x = BatchNormalization()(x)\n  x = Dense(150, activation='relu')(x)\n  x = BatchNormalization()(x)\n  x = Dropout(0.5)(x)\n  x = Dense(100, activation='relu')(x)\n  x = BatchNormalization()(x)\n  x = Dropout(0.2)(x)\n  x = Dense(50, activation='relu')(x)\n  x = BatchNormalization()(x)\n  x = Dropout(0.1)(x)\n  x = Dense(1)(x)\n  nn = Model([store_in,item_in,cluster_in,day_in,city_in,family_in,type_in,state_in,class_in,onpromotion_in,dcoilwtico_in], x)\n  nn.compile(optimizer='adam', loss='mse',metrics=['mae'])\n  nn.summary()\n  return nn\n\ndef train_embedding(X,y,X_val,y_val, better_model=False):\n  \"\"\"\n  train a NN model \n  \"\"\"\n\n  callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n  if not better_model:\n    embedding_model = get_embedding_model()\n  else:\n    embedding_model = get_embedding_model_2()  \n  log_dir = '/kaggle/working/'+\"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n  tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n  history = embedding_model.fit(X, y, batch_size=512, epochs=10, validation_data=(X_val, y_val),callbacks=[callback,tensorboard_callback])\n\n                           \n\n  return embedding_model, history","metadata":{"id":"aj1XblM4hfce","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3d. Predected targets using only the features embeddings in a DL regressor","metadata":{"id":"39A4qW-hbh2Y"}},{"cell_type":"code","source":"embedding_model_1, history_1 = train_embedding(x_train_s,Y_train_s,x_val_s,Y_val_s)","metadata":{"id":"M-QtQZzGZA0r","outputId":"daef0776-7781-4628-c592-e3d84f622d09","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%tensorboard --logdir ./logs","metadata":{"id":"G-ceae6lDZ64","outputId":"769bca7c-e95b-4b6c-adad-6ce1e0770c92","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, x):\n    y = model.predict(x, batch_size=128, verbose=1, workers=4, use_multiprocessing=True)\n    y = y.reshape(-1)\n    y = np.clip(y, 0, max(0, y.max()))\n    return y\n\npredictions = predict(embedding_model_1, x_val_s)\nscore = NWRMSLE(Y_val_s, predictions, wightes_validtion.values)\nprint('NWRMSLE VALIDATION:',score)","metadata":{"id":"OkKlsDOOyDeQ","outputId":"deb8d0e7-ccc2-4db2-e2c1-9f89ef212a3d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_to_csv_test(df_test,embedding_model,name, complex_m=False):\n\n  def simple_to_Xy_array(df):\n    features = ['store_nbr',\n                'item_nbr',\n                'cluster',\n                'day']\n\n    X = []\n    for f in features:\n        X.append(df[f])\n    return X\n\n  def to_Xy_array_csv(df):\n    features = ['store_nbr',\n                'item_nbr',\n                'cluster',\n                'day',\n                'city',\n                'family',\n                'type',\n                'state',\n                'class',\n                'onpromotion',\n                'dcoilwtico']\n\n    target_class = 'unit_sales'\n\n    X = []\n\n    for f in features:\n        X.append(df[f])\n\n    return X\n\n  def save_test_csv(model,X_test,name):\n\n        y_test_tag = model.predict(X_test,verbose=1)\n        y_test_tag = np.clip(y_test_tag, 0, max(0, y_test_tag.max()))\n        aaa = y_test_tag.reshape(1,-1)\n        predictions = pd.DataFrame({'id': test['id'].values, 'unit_sales': aaa[0]})\n        predictions.to_csv('/kaggle/working/model_'+name+'.csv', index=False)\n\n\n  # df_test, wightes_test\n  if not complex_m:\n    x_test = simple_to_Xy_array(df_test)\n  else:\n     x_test = to_Xy_array_csv(df_test)\n  y_test_tag = save_test_csv(embedding_model,x_test,name)\n\nsave_to_csv_test(df_test,embedding_model_1,'1',complex_m=False)","metadata":{"id":"39TjUkF2KKaM","outputId":"fc9f6007-eeb3-4041-d18a-aacffea30668","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3d Fitting second model","metadata":{"id":"poyl5gHbl3tN"}},{"cell_type":"code","source":"embedding_model_2, history_2 = train_embedding(x_train,Y_train,x_val,Y_val,better_model=True)","metadata":{"id":"FoG0NRO_8RzS","outputId":"a9b4ace4-430a-4b77-cf4b-f10d4669bd49","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = predict(embedding_model_2, x_val)\nscore = NWRMSLE(Y_val_s, predictions, wightes_validtion.values)\nprint('NWRMSLE VALIDATION:',score)","metadata":{"id":"mN6FH8eFsY_5","outputId":"ac0e357b-46d2-43c1-fdc6-35ceb3520020","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_to_csv_test(df_test,embedding_model_2,'2',complex_m=True)","metadata":{"id":"PqIdzMKXEHWQ","outputId":"cc965936-b2fc-49d0-b9da-e9471f47057c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3f. Insights from the embeddings of categorical features\n\nWe are plotting the embeddings of store and item ids","metadata":{"id":"hW33ppMYGZzC"}},{"cell_type":"code","source":"from sklearn.manifold import TSNE\n\nweights = embedding_model_1.get_weights()\nstore_embeddings = weights[0]\nitem_embeddings = weights[1]\n\ndef display_wordlist(store_enc, store_embeddings,name):\n    vectors = [store_embeddings[word-1] for word in store_enc]\n    word_labels = [word for word in store_enc]\n    word_vec_zip = zip(word_labels, vectors)\n\n    # Convert to a dict and then to a DataFrame\n    word_vec_dict = dict(word_vec_zip)\n    df = pd.DataFrame.from_dict(word_vec_dict, orient='index')\n\n    # Use tsne to reduce to 2 dimensions\n    tsne = TSNE(perplexity=65,n_components=2, random_state=0)\n    np.set_printoptions(suppress=True)\n    Y = tsne.fit_transform(df)\n\n    x_coords = Y[:, 0]\n    y_coords = Y[:, 1]\n    # display plot\n    plt.figure(figsize=(16, 8)) \n    plt.plot(x_coords, y_coords, 'ro')\n\n    for label, x, y in zip(df.index, x_coords, y_coords):\n        plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points')\n    plt.xlim(x_coords.min()+0.00005, x_coords.max()+0.00005)\n    plt.ylim(y_coords.min()+0.00005, y_coords.max()+0.00005)\n    plt.show()\n\ndisplay_wordlist(store_enc,store_embeddings,'store')\ndisplay_wordlist(item_enc,item_embeddings,'item')","metadata":{"id":"J_lCfTqtGY-2","outputId":"7552a9ad-076a-437d-e05c-40332a42499c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can notice that there is no significant difference between stores in opposite to the items where we can notice that there are several groups of items. This results are corollated with SHAP results.","metadata":{}},{"cell_type":"markdown","source":"# 3g. Feature extractor” for a classical ML algorithm \n\nWe extraced 50 feturess from the last Danse layer from the first model and used them as input to catboost model.","metadata":{"id":"yPLvnMHuoG9A"}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn import metrics\nfrom sklearn.metrics import classification_report\nimport time\n      \n      \nextract = Model(embedding_model_1.inputs, embedding_model_1.layers[-3].output)\nfeatures = extract.predict(x_train_s)\nfeatures_test = extract.predict(x_val_s)\n\n# Generate a print\nprint('------------------------------Cat Boost--------------------------------')\n\n# clf=RandomForestRegressor(max_depth=features.shape[1], n_jobs=-1, random_state=42, verbose=2)\nclf = CatBoostRegressor()\n\n# measure time to train model:\nstart = time.time()\n\n#Train the model using the training sets y_pred=clf.predict(X_test)\nclf.fit(features,Y_train_df)  \n\nend = time.time()\nprint(f' time for pretrained model to train: {end - start}')\n\ny_pred=clf.predict(features_test)\n\n# Model Accuracy, how often is the classifier correct?\nscore = NWRMSLE(Y_val_df, y_pred, wightes_validtion.values)\nprint('NWRMSLE:',score)\n\n\n\n","metadata":{"id":"hFmO3up-mCYU","outputId":"c2e997c4-f6d9-4aaf-92a4-c3ea9560015b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = NWRMSLE(Y_val_df, y_pred, wightes_validtion.values)\nprint('NWRMSLE:',score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}