{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install py7zr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{},"cell_type":"markdown","source":"imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Input, Embedding, dot, add, Flatten, concatenate, Dropout, Dense, BatchNormalization\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import plot_model \nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.decomposition import PCA\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport pickle\nimport py7zr\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"extract all data"},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        path = os.path.join(dirname, filename)\n        print(path)\n        with py7zr.SevenZipFile(path, mode='r') as z:\n            z.extractall()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"read all tables"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('oil')\noil = pd.read_csv('./oil.csv', parse_dates=[\"date\"])\nprint('items')\nitems = pd.read_csv('./items.csv')\nprint('stores')\nstores = pd.read_csv('./stores.csv')\nprint('test')\ntest = pd.read_csv('./test.csv', parse_dates=[\"date\"])\nprint('train')\ntrain = pd.read_csv('./train.csv', usecols=[1, 2, 3, 4, 5], parse_dates=[\"date\"],\n                     converters={'unit_sales': lambda u: u if float(u) > 0 else 0},\n                     skiprows = range(1, 118968557+1)) # read from 2017-06-15 to the end of thrain data- 2017-08-14\ntrain.unit_sales = train.unit_sales.astype(pd.np.float64)\nprint('validation')\nvalidation = pd.read_csv('./train.csv', usecols=[1, 2, 3, 4, 5], parse_dates=[\"date\"],\n                     converters={'unit_sales': lambda u: u if float(u) > 0 else 0},\n                     skiprows = range(1, 115675948+1), nrows=118968556-115675948+1) # read from 2017-05-15 to 2017-06-14\nvalidation.unit_sales = validation.unit_sales.astype(pd.np.float64)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data preperation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And on the target variable"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(train['unit_sales']).describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we want to merge all relevant data to one dataset.\nAfter looking at the different tables, we chose to merge the oil, stores and items tables with the main tables- train, test and validation. We chose to ignore from the data in the holidays_events table because even though it may help with the problem, it is too complicated to take it under concideration.\nAlso, we didn't took the data from transactions table because in the testing we dont have this data availble."},{"metadata":{},"cell_type":"markdown","source":"In order to merge the oil pricing data, we will firstly fill it with all the missing dates, and fill all missing prices with moving avarage. "},{"metadata":{"trusted":true},"cell_type":"code","source":"dates = train.date\ndates = dates.append(test.date)\ndates = dates.append(validation.date)\nunique_dates = pd.DataFrame({'date': dates.unique()})\n\noil = oil.merge(unique_dates, on='date', how='outer')\noil = oil.sort_values('date')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oil = oil.fillna(oil['dcoilwtico'].rolling(10, min_periods=1, center=True, win_type='gaussian')\n                 .mean(std=1).to_frame())\nprint(len(oil[oil['dcoilwtico'].isna()]) == 0)\nprint(oil.shape)\nprint(oil[oil['dcoilwtico'].isna()].shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look at the prices plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(x=oil['date'], y=oil['dcoilwtico'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we will orgenize all item numbers to be between 0 to unique items amount-1: the items_nbr are not in order so we will fix it by adding an items encoding column\nWe will also fix the stores number to be also between 0 to unique stores amount-1, and encode all other features"},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\n\n\nstores['store_enc']   = le.fit_transform(stores['store_nbr'].values)\nstores['cluster_enc'] = le.fit_transform(stores['cluster'].values)\nstores['state_enc']   = le.fit_transform(stores['state'].values)\nstores['type_enc']    = le.fit_transform(stores['type'].values)\nstores['city_enc']    = le.fit_transform(stores['city'].values)\n\nitems['item_enc']   = le.fit_transform(items['item_nbr'].values)\nitems['family_enc'] = le.fit_transform(items['family'].values)\nitems['class_enc'] = le.fit_transform(items['class'].values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And now we can merge all tables together."},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_data(df):\n    print('1. original', df.shape)\n    df = df.merge(oil, on='date', how='left')\n    print('2. oil', df.shape)\n    df = df.merge(stores, on='store_nbr', how='left')\n    print('3. stores', df.shape)\n    df = df.merge(items, on='item_nbr', how='left')\n    print('4. items', df.shape)\n#     df = df.merge(transactions, on=['date', 'store_nbr'], how='left')\n\n    df['onpromotion'] = df['onpromotion'].fillna(False) # fill all unavailble data for promotion with default false\n    \n    df['onpromotion'] = df['onpromotion'].astype('category').cat.codes\n    df['perishable'] = df['perishable'].map({0:1.0, 1:1.25})\n    df['day_enc'] = df.date.dt.dayofweek\n    \n    df = df.drop(['store_nbr', 'item_nbr', 'city','state', 'type', 'cluster', 'family', 'class'], axis=1)\n    \n\n    \n    # check for any NaNs in data- shouldn't be any at this point\n    for col in df.columns:\n        nans = len(df[df[col].isna()])\n        if nans > 0:\n            print(col, nans)\n    print('5. finally', df.shape)\n\n    return df\n\nprint('train')\ntrain = merge_data(train)\nprint('**************')\nprint('test')\ntest = merge_data(test)\nprint('**************')\nprint('validation')\nvalidation = merge_data(validation)\nprint('**************')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's see all the uinque amount for each feature in the data."},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in train.columns:\n    print(len(train[col].unique()), col)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And the correlation between the features and the target variable- unit_sales"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(figsize=(10,10))\nhm = sns.heatmap(train.corr(), ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see in the correlation matrix, there are no two variables that have high correlation.\n"},{"metadata":{},"cell_type":"markdown","source":"Now let's look on the plots of each feature vs the target variable."},{"metadata":{"trusted":true},"cell_type":"code","source":"features = len(train.columns)\nfig, sub = plt.subplots(features, figsize=(20,50))\n\nfor i in range(features):\n    col = train.columns[i]\n    sub[i].scatter(x=train[col], y=train['unit_sales'])\n    sub[i].set_title(str(col) + ' vs unit_sales')\n    sub[i].set_xlabel(col)\n    sub[i].set_ylabel('unit_sales')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We didn't find anything interesting in the data plots."},{"metadata":{"trusted":true},"cell_type":"code","source":"def NWRMSLE(y, pred, w):\n#     return mean_squared_error(y, pred, sample_weight=w)**0.5\n    y = np.array(y)\n    pred = np.array(pred)\n    w = np.array(w)\n    \n    return (((w*(np.log(pred+1) - np.log(y+1)))**2).sum() / w.sum())**0.5\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_model(model, model_name, X_val, y_val, save=False,\n                   X_train=None, y_train=None, y_val_tag=None, X_test=None, y_test_tag=None):\n    #fitting\n    if X_train is not None and y_train is not None:\n        print(\"fitting....\")\n        model.fit(X_train, y_train)\n    if save:\n        pickle.dump(model, open(model_name+'.h5', 'wb'))\n    \n    # validation\n    if y_val_tag is None:\n        print('predict validation....')\n        y_val_tag = model.predict(X_val)\n        y_val_tag = np.clip(y_val_tag, 0, max(0, y_val_tag.max()))\n\n    val = pd.DataFrame({'id': range(len(y_val)),\n                        'date': validation.date, \n                        'real': y_val, \n                        'pred': y_val_tag})\n    \n    print('validation NWRMSLE = %f' %(NWRMSLE(val.real, val.pred, validation.perishable)))\n    \n    plt.figure(figsize = (16,9))\n    plt.scatter(val.real, val.pred)\n    plt.title(\"real vs pred\")\n    plt.xlabel(\"real\")\n    plt.ylabel(\"pred\")\n    plt.show()\n\n    plt.figure(figsize = (16,9))\n    plt.plot('id', 'real', data=val, marker='o', markerfacecolor='blue', markersize=5, color='blue', linewidth=2)\n    plt.plot('id', 'pred', data=val, marker='x', markerfacecolor='red', markersize=5, color='red', linewidth=2)\n    plt.xlabel(\"id\")\n    plt.ylabel(\"unit_sales\")\n    plt.legend()\n    \n    # test\n    if X_test is not None or y_test_tag is not None:\n        print(\"in test\")\n        if y_test_tag is None:\n            print(\"predicting test.....\")\n            y_test_tag = model.predict(X_test)\n            y_test_tag = np.clip(y_test_tag, 0, max(0, y_test_tag.max()))\n        test_predictions = pd.DataFrame({'id': test['id'], 'unit_sales': y_test_tag})\n        test_predictions.to_csv('test_preds_'+model_name+'.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.b"},{"metadata":{},"cell_type":"markdown","source":"those are the features we will look at in oure classical ML model"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['onpromotion', 'dcoilwtico', 'store_enc',\n            'type_enc', 'city_enc', 'state_enc', 'cluster_enc',\n            'class_enc','item_enc', 'family_enc', 'day_enc']\nX_train = train[features]\nX_val = validation[features]\nX_test = test[features]\n\ny_train = train['unit_sales']\ny_val = validation['unit_sales']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we chose to use Random forest regressor as our classical ML- we tried several other algorithms such as SVM and lightGBM. SVM tool alot of time to fit, and lgbm gave really bad results so we chose RF"},{"metadata":{"trusted":true},"cell_type":"code","source":"rfrm = RandomForestRegressor(max_depth=X_train.shape[1], n_jobs=-1, random_state=0, verbose=2)\nrfrm.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(rfrm, 'rfrm', X_val, y_val, X_test=X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://i.imgur.com/YAAf8HS.png)"},{"metadata":{},"cell_type":"markdown","source":"As we can see, RF gave avarage results"},{"metadata":{},"cell_type":"markdown","source":"Let's try to look at the features importance to the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"importance = rfrm.feature_importances_\n\n# summarize feature importance\nfor i,v in enumerate(importance):\n\tprint('Feature: %s, Score: %.5f' % (features[i],v))\n    \n# plot feature importance\nplt.figure(figsize = (16,9))\nplt.bar([features[x] for x in range(len(importance))], importance)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3.c\n\nMost of the relevant pre-processing done above in the data preperation section."},{"metadata":{},"cell_type":"markdown","source":"# 3.d"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_x_y(df, features, target=None):\n    X = []\n    for f in features:\n        X.append(df[f])\n    if target is not None:    \n        y = df[target]\n        return X, y\n    return X","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"features for the embedding model- which features we will use for embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['class_enc', 'onpromotion', 'store_enc', 'type_enc',\n                'city_enc', 'state_enc', 'cluster_enc',\n                'item_enc', 'family_enc', 'day_enc']\ntarget = 'unit_sales'\nX_train, y_train = get_x_y(train, features, target)\nX_val, y_val = get_x_y(validation, features, target)\nX_test = get_x_y(test, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def embedding_input(name, n_in, n_out, reg=1e-4):\n    inp = Input(shape=(1,), dtype='int64', name=name+'_in')\n    return inp, Embedding(n_in, n_out, input_length=1, activity_regularizer=l2(reg), name=name+'_emb')(inp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"model building"},{"metadata":{"trusted":true},"cell_type":"code","source":"class_in, class_emb = embedding_input('class', len(items['class_enc'].unique()), 20)          # 20/337\npromotion_in, promotion_emb = embedding_input('promotion', 2, 1)                              # 1/2\nstore_in, store_emb = embedding_input('store', len(stores['store_enc'].unique()), 15)         # 15/54\ntype_in, type_emb = embedding_input('type', len(stores['type_enc'].unique()), 3)              # 3/5\ncity_in, city_emb = embedding_input('city', len(stores['city_enc'].unique()), 7)              # 7/22\nstate_in, state_emb = embedding_input('state', len(stores['state_enc'].unique()), 5)          # 5/16\ncluster_in, cluster_emb = embedding_input('cluster', len(stores['cluster_enc'].unique()), 5)  # 5/17\nitem_in, item_emb = embedding_input('item', len(items['item_enc'].unique()), 50)              # 50/4100\nfamily_in, family_emb = embedding_input('family', len(items['family_enc'].unique()), 15)      # 15/33\nday_in, day_emb = embedding_input('day', 7, 4)                                                # 4/7\n\n\nx = concatenate([class_emb, promotion_emb, store_emb, type_emb, city_emb, state_emb,\n                 cluster_emb, item_emb, family_emb, day_emb])\nx = Flatten()(x)\nx = BatchNormalization()(x)\nx = Dense(32, activation='relu')(x)\nx = Dense(16, activation='relu')(x)\nx = Dropout(0.5)(x)\nx = Dense(1)(x)\n\nemb_model = Model([class_in, promotion_in, store_in,\n                   type_in, city_in, state_in, cluster_in,\n                   item_in, family_in, day_in], x)\nemb_model.compile(optimizer='adam', loss=['mse'], metrics=['mae'])\nemb_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(emb_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"emb_model.fit(X_train,y_train, batch_size=2**10, epochs=5, verbose=1,\n              validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, x):\n    y = model.predict(x, batch_size=128, verbose=1, workers=4, use_multiprocessing=True)\n    y = y.reshape(-1)\n    y = np.clip(y, 0, max(0, y.max()))\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_val_tag = predict(emb_model, X_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_tag = predict(emb_model, X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(emb_model, 'embedding', X_val, y_val,\n               y_val_tag=y_val_tag, y_test_tag=y_test_tag)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://i.imgur.com/U8Xljfu.png)"},{"metadata":{},"cell_type":"markdown","source":"as we can see the embedding model gave us slightly better results, but still only avarage"},{"metadata":{},"cell_type":"markdown","source":"# 3.e"},{"metadata":{},"cell_type":"markdown","source":"features for our bigger network- we will try to add the oil prices as feature to the embedding network from 3.d"},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['dcoilwtico', 'class_enc', 'onpromotion', 'store_enc',\n            'type_enc', 'city_enc', 'state_enc', 'cluster_enc',\n            'item_enc', 'family_enc', 'day_enc']\ntarget = 'unit_sales'\nX_train, y_train = get_x_y(train, features, target)\nX_val, y_val = get_x_y(validation, features, target)\nX_test = get_x_y(test, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oil_in = Input(shape=(1,1), dtype='float64', name='oil_in')\n\nclass_in, class_emb = embedding_input('class', len(items['class_enc'].unique()), 20)          # 20/337\npromotion_in, promotion_emb = embedding_input('promotion', 2, 1)                              # 1/2\nstore_in, store_emb = embedding_input('store', len(stores['store_enc'].unique()), 15)         # 15/54\ntype_in, type_emb = embedding_input('type', len(stores['type_enc'].unique()), 3)              # 3/5\ncity_in, city_emb = embedding_input('city', len(stores['city_enc'].unique()), 7)              # 7/22\nstate_in, state_emb = embedding_input('state', len(stores['state_enc'].unique()), 5)          # 5/16\ncluster_in, cluster_emb = embedding_input('cluster', len(stores['cluster_enc'].unique()), 5)  # 5/17\nitem_in, item_emb = embedding_input('item', len(items['item_enc'].unique()), 50)              # 50/4100\nfamily_in, family_emb = embedding_input('family', len(items['family_enc'].unique()), 15)      # 15/33\nday_in, day_emb = embedding_input('day', 7, 4)                                                # 4/7\n\n\nx = concatenate([oil_in, class_emb, promotion_emb, store_emb, type_emb, city_emb, state_emb,\n                 cluster_emb, item_emb, family_emb, day_emb])\nx = Flatten()(x)\nx = BatchNormalization()(x)\nx = Dense(32, activation='relu')(x)\nx = Dense(16, activation='relu')(x)\nx = Dropout(0.5)(x)\nx = Dense(1)(x)\n\nmodel = Model([oil_in, class_in, promotion_in, store_in,\n                   type_in, city_in, state_in, cluster_in,\n                   item_in, family_in, day_in], x)\nmodel.compile(optimizer='adam', loss=['mse'], metrics=['mae'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_train,y_train, batch_size=2**10, epochs=5, verbose=1,\n              validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_val_tag = predict(model, X_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_tag = predict(model, X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(model, '3e', X_val, y_val,\n               y_val_tag=y_val_tag, y_test_tag=y_test_tag)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://i.imgur.com/bdVIz3e.png)"},{"metadata":{},"cell_type":"markdown","source":"as we can see the oil prices addition improved our model slightly, but not so much"},{"metadata":{},"cell_type":"markdown","source":"# 3.f"},{"metadata":{},"cell_type":"markdown","source":"we will look at the embedding of the store numbers, compared to the original store clustering recived in store dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"emb_store_model = Model(inputs=emb_model.get_layer('store_in').input,\n                        outputs=emb_model.get_layer('store_emb').output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clusters_colors = {0 : '#FF0000',\n                   1 : '#FF00DE',\n                   2 : '#9A00FF',\n                   3 : '#2B00FF',\n                   4 : '#00A2FF',\n                   5 : '#00FF3C',\n                   6 : '#E6FF00',\n                   7 : '#FFCD00',\n                   8 : '#981919',\n                   9 : '#199865',\n                   10: '#984719',\n                   11: '#EB5A5A', \n                   12: '#815AEB',\n                   13: '#5AEB81',\n                   14: '#6D6F6E',\n                   15: '#97B77B',\n                   16: '#33610C'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca = PCA(n_components=2)\ns = emb_store_model.predict(stores.store_enc.unique())\npca.fit(np.squeeze(s))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,8))\n\nfor cluster in range(len(stores.cluster_enc.unique())):\n#     print(cluster)\n    strs = stores[stores['cluster_enc']==cluster].store_enc\n    store_embeddings = emb_store_model.predict(strs)\n    \n    store_embeddings = np.squeeze(store_embeddings)\n    if len(store_embeddings.shape) < 2:\n        store_embeddings=store_embeddings.reshape(-1,1)\n    store_embeddings = pca.transform(store_embeddings)\n\n    x_plt = store_embeddings[:,0]\n    y_plt = store_embeddings[:,1]\n    \n    plt.scatter(x_plt,y_plt, c=clusters_colors[cluster], label=cluster)\nplt.legend(loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see in the figure above, all the stores in clusters 16, 4, 15 aligned in one line- which might suggest the model was able to learn good embedding to part of the stores, but not to others (like cluster 2 which is very scatterd)."},{"metadata":{},"cell_type":"markdown","source":"# 3.g","attachments":{}},{"metadata":{},"cell_type":"markdown","source":"the features we will use for feature extraction "},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['class_enc', 'onpromotion', 'store_enc', 'type_enc',\n                'city_enc', 'state_enc', 'cluster_enc',\n                'item_enc', 'family_enc', 'day_enc']\ntarget = 'unit_sales'\nX_train, y_train = get_x_y(train, features, target)\nX_val, y_val = get_x_y(validation, features, target)\nX_test = get_x_y(test, features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will ture the embedding model from 3.d right after the flatten layer to recive all of the new features produced by the embeddings."},{"metadata":{"trusted":true},"cell_type":"code","source":"fe_model = Model(emb_model.input, emb_model.layers[-6].output)\nfe_model.compile(optimizer='adam', loss=['mse'], metrics=['mae'])\nfe_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(fe_model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And now to create the new dataset- we will feed the network with all our data to recive the new features"},{"metadata":{"trusted":true},"cell_type":"code","source":"fe_x_train = fe_model.predict(X_train, batch_size=128, verbose=1, workers=4, use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\ndel train\ndel X_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fe_x_val = fe_model.predict(X_val, batch_size=128, verbose=1, workers=4, use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fe_x_test = fe_model.predict(X_test, batch_size=128, verbose=1, workers=4, use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After some tries on different algorithms like SVM and RF which took too much time to run, and SGD which took too much memory which we didnt have at this point, we used the LGB  model."},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\n# create dataset for lightgbm\nlgb_train = lgb.Dataset(fe_x_train, y_train)\nlgb_eval = lgb.Dataset(fe_x_val, y_val, reference=lgb_train)\n\n# specify your configurations as a dict\nparams = {\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': {'l2', 'l1'},\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0\n}\n\nprint('Starting training...')\n# train\ngbm = lgb.train(params,\n                lgb_train,\n                num_boost_round=30,\n                valid_sets=lgb_eval,\n                early_stopping_rounds=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_tag_val = gbm.predict(fe_x_val, num_iteration=gbm.best_iteration)\ny_tag_val = np.clip(y_tag_val, 0, max(0, y_tag_val.max()))\n\ny_tag_test = gbm.predict(fe_x_test, num_iteration=gbm.best_iteration)\ny_tag_test = np.clip(y_tag_test, 0, max(0, y_tag_test.max()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_model(gbm, 'fe_gbm', fe_x_val, y_val,\n                   y_val_tag=y_tag_val, y_test_tag=y_tag_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](https://i.imgur.com/lHAa2qi_d.webp?maxwidth=760&fidelity=grand)"},{"metadata":{},"cell_type":"markdown","source":"As we can see this model gave us the worst results so far."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}