{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.regularizers import l2\nfrom keras.optimizers import *\nfrom keras.utils import to_categorical\nimport datetime\nfrom sklearn.preprocessing import LabelEncoder\nimport gc\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom keras import backend as K\nfrom sklearn.model_selection import KFold\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86cae6aa917478a27cd397d7dcc949447e696e5e"},"cell_type":"markdown","source":"# Define global functions"},{"metadata":{"trusted":true,"_uuid":"4f4a12768f717d1dbf1671bdae5021816b60dd1e"},"cell_type":"code","source":"# define function that calaculate RMSE\ndef root_mean_squared_error(y_true, y_pred):\n        return K.sqrt(K.mean(K.square(y_pred - y_true)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9f7ce14f675a7fb562e70187ca409b14db27d3e"},"cell_type":"markdown","source":"# Read data"},{"metadata":{"trusted":true,"_uuid":"6b5c30df1f3fc70b60de8261c2b13fa58ac0b0e0"},"cell_type":"code","source":"train_set = pd.read_csv(\"../input/bgu-dl-assignmnt2-features-extraction/train_set.csv\")\ntest_set = pd.read_csv(\"../input/bgu-dl-assignmnt2-features-extraction/test_set.csv\")\ntarget = pd.read_csv(\"../input/bgu-dl-assignmnt2-features-extraction/target.csv\", header=None)\n\nprint(\"shape of train : \",train_set.shape)\nprint(\"shape of test : \",test_set.shape)\nprint(\"shape of target : \",target.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17cc95098cb7021f7c27e06377faa737a691e609"},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":true,"_uuid":"19424a660e66e4e32233e51cf7f6b7b3231e1273"},"cell_type":"code","source":"cat_col = ['feature_1','feature_2', 'feature_3', 'merchant_group_id', 'merchant_category_id', 'subsector_id', 'category_1',\n          'most_recent_sales_range', 'most_recent_purchases_range', 'category_4', 'city_id', 'state_id', 'category_2']\nnumeric_col = train_set.columns[~train_set.columns.isin(np.append(cat_col, ['card_id', 'first_active_month']))]\nused_col = np.concatenate((cat_col, numeric_col), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c47482796675c87a0438f4a6a96a801d3482c901"},"cell_type":"code","source":"def preprocess(trx_data):\n    for cat_col_name in cat_col:\n        lbl = LabelEncoder()\n        lbl.fit(trx_data[cat_col_name].unique().astype('str'))\n        trx_data[cat_col_name] = lbl.transform(trx_data[cat_col_name].astype('str'))\n    \n    for numeric_col_name in numeric_col:\n        trx_data[numeric_col_name] = pd.to_numeric(trx_data[numeric_col_name])\n        min_val = trx_data[numeric_col_name].min()\n        max_val = trx_data[numeric_col_name].max()\n        if min_val == max_val:\n            trx_data[numeric_col_name] = 0\n            print(numeric_col_name)\n        else:\n            trx_data[numeric_col_name] = (max_val - trx_data[numeric_col_name]) / (max_val - min_val)\n\n    return trx_data\n\n# remove nan values from data set\ntrain_set_no_nan = train_set.fillna(-20)\ntest_set_no_nan = test_set.fillna(-20)\n\ntrain_set = preprocess(train_set_no_nan)\ntest_set = preprocess(test_set_no_nan)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0eaf388d25ace39a6f781d532ead2e802ff78946"},"cell_type":"markdown","source":"# split the given train set to train and test set"},{"metadata":{"trusted":true,"_uuid":"b5f97a5ce3daac2ef4caf508bd4692a2d6cd67d6"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_set, target, test_size=0.2, random_state=24)\n\nprint('X_train shape', X_train.shape)\nprint('X_test shape', X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be17a6f30efd1d8214606b2ef154b21f6ef3b431"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d030c05a5d135ef44fb12e0f1163d8eedd895ec2"},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"565f7c63304115cdc8a6aaa3c5cddadd0b3c5ed2"},"cell_type":"markdown","source":"# Classic ML model to form a benchmark"},{"metadata":{"trusted":true,"_uuid":"38e8bd0683b71e7af9264785787f34782e79ff8a"},"cell_type":"code","source":"lgb_model = lgb.LGBMRegressor()\nlgb_model.fit(X_train[used_col], y_train.iloc[:,0])\n\nlgb_pred_test = lgb_model.predict(X_test[used_col])\nlgb_pred_train = lgb_model.predict(X_train[used_col])\n\nprint('test RMSE:', mean_squared_error(y_test, lgb_pred_test) ** 0.5)\nprint('train RMSE:', mean_squared_error(y_train, lgb_pred_train) ** 0.5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cefaf989f7496b60442ec4b32b278edc3b14410"},"cell_type":"markdown","source":"# Create embedding to categorical feature"},{"metadata":{"trusted":true,"_uuid":"aa090faf76ffe84898296093d79e0380f51b2d6d"},"cell_type":"code","source":"embedding_col = ['feature_1', 'feature_2', 'feature_3', 'year', 'month']\nother_col = [x for x in used_col if x not in embedding_col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43185616539bf08f3eebd2c80a306940f23088b7"},"cell_type":"code","source":"f1_unique_val = len(train_set['feature_1'].unique())\nf2_unique_val = len(train_set['feature_2'].unique())\nf3_unique_val = len(train_set['feature_3'].unique())\nyear_unique_val = len(train_set['year'].unique())\nmonth_unique_val = len(train_set['month'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71ee8991faa513ee2ebe54ff0335848cd73be0a0"},"cell_type":"code","source":"f1_inp = Input(shape=(1,),dtype='int64')\nf2_inp = Input(shape=(1,),dtype='int64')\nf3_inp = Input(shape=(1,),dtype='int64')\nyear_inp = Input(shape=(1,),dtype='int64')\nmonth_inp = Input(shape=(1,),dtype='int64')\n\nf1_emb = Embedding(f1_unique_val,2,input_length=1, embeddings_regularizer=l2(1e-6))(f1_inp)\nf2_emb = Embedding(f2_unique_val,1,input_length=1, embeddings_regularizer=l2(1e-6))(f2_inp)\nf3_emb = Embedding(f3_unique_val,1,input_length=1, embeddings_regularizer=l2(1e-6))(f3_inp)\nyear_emb = Embedding(year_unique_val,3,input_length=1, embeddings_regularizer=l2(1e-6))(year_inp)\nmonth_emb = Embedding(month_unique_val,4,input_length=1, embeddings_regularizer=l2(1e-6))(month_inp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b11848ae55fe9b65848541054fa47d90c913b8a6"},"cell_type":"markdown","source":"# Predicting the target using only the categorical features embeddings"},{"metadata":{"trusted":true,"_uuid":"fa07722e76be13e0f6cd975da0d900b2d01d5c12"},"cell_type":"code","source":"x = concatenate([f1_emb,f2_emb,f3_emb,year_emb,month_emb])\nx = Flatten()(x)\nx = BatchNormalization()(x)\nx = Dense(10,activation='relu')(x)\nx = Dense(10,activation='relu')(x)\nx = Dropout(0.4)(x)\nx = BatchNormalization()(x)\nx = Dense(10,activation='relu')(x)\nx = Dense(10,activation='relu')(x)\nx = Dropout(0.7)(x)\nx = Dense(1, activation='sigmoid')(x) #activation='linear'\nemb_model = Model([f1_inp,f2_inp,f3_inp,year_inp,month_inp],x)\n#emb_model.compile(loss='mse',optimizer='adam')\n\nemb_model.compile(optimizer=\"RMSProp\", loss=root_mean_squared_error)\n\nprint(emb_model.summary())\n\nemb_model.fit([X_train[col] for col in embedding_col], y_train, epochs=5)\n\nemb_pred_test = emb_model.predict([X_test[col] for col in embedding_col])\nemb_pred_train = emb_model.predict([X_train[col] for col in embedding_col])\n\nprint('test RMSE', mean_squared_error(y_test, emb_pred_test) ** 0.5)\nprint('train RMSE', mean_squared_error(y_train, emb_pred_train) ** 0.5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8991d93bb0e46777090170c7b8e3669d7a2cc490"},"cell_type":"markdown","source":"# Use the embeddings we got as a “feature extractor” for a LGBMRegressor model"},{"metadata":{"trusted":true,"_uuid":"181de76369ffd9f6e102904f355c0a59cfdb9b38"},"cell_type":"code","source":"emb_output = emb_model.layers[11].output\n\nfeature_model = Model(emb_model.input, emb_output)\n\nfeature_model.compile(optimizer = \"RMSProp\", loss = root_mean_squared_error)\nprint(feature_model.summary())\n\nfeaturs = feature_model.predict([X_train[col] for col in embedding_col])\nfeatures_test = feature_model.predict([X_test[col] for col in embedding_col])\n\nlgb_model = lgb.LGBMRegressor()\nlgb_model.fit(featurs, y_train.values)\n\nlgb_pred_test = lgb_model.predict(features_test)\nlgb_pred_train = lgb_model.predict(featurs)\n\nprint('test RMSE', mean_squared_error(y_test, lgb_pred_test) ** 0.5)\nprint('train RMSE', mean_squared_error(y_train, lgb_pred_train) ** 0.5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f9502dc4a042bb0c1e4eb1026ab1ba72fa582e1"},"cell_type":"markdown","source":"# Add the rest of the features "},{"metadata":{"trusted":true,"_uuid":"093c9733924a4ec78a336587be18fa3791dcac6b"},"cell_type":"code","source":"# define continuous input\ncontinuous_input = Input(shape=(len(other_col),))\n\n# define categorical input                         \nf1_emb = Reshape((2,))(f1_emb)\nf2_emb = Reshape((1,))(f2_emb)\nf3_emb = Reshape((1,))(f3_emb)\nyear_emb = Reshape((3,))(year_emb)\nmonth_emb = Reshape((4,))(month_emb)\n                         \n#split train set to train and validation set\nX_train_val, X_val, y_train_val, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=6)\n                         \n# define function to create input to model\ndef get_input(data):\n    inp = [data[other_col], data['feature_1'], data['feature_2'], data['feature_3'], data['year'], data['month']]\n    return inp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91e12c16b437a35c5cc063095121613e18ce343a"},"cell_type":"code","source":"x = concatenate([continuous_input,f1_emb,f2_emb,f3_emb,year_emb,month_emb])\nx = BatchNormalization()(x)\nx = Dense(10,activation='relu')(x)\nx = Dense(10,activation='relu')(x)\nx = Dropout(0.4)(x)\nx = BatchNormalization()(x)\nx = Dense(10,activation='relu')(x)\nx = Dense(10,activation='relu')(x)\nx = Dropout(0.7)(x)\nx = Dense(1, activation='linear')(x)\nemb_cont_model = Model([continuous_input,f1_inp,f2_inp,f3_inp,year_inp,month_inp],x)\n\nrmsprop_opt = RMSprop(lr=0.005)\nemb_cont_model.compile(optimizer = rmsprop_opt, loss = root_mean_squared_error)\n\nprint(emb_cont_model.summary())\n\n'''def set_callbacks(description='run1',patience=15,tb_base_logdir='./logs/'):\n    cp = ModelCheckpoint('best_model_weights_{}.h5'.format(description),save_best_only=True)\n    es = EarlyStopping(patience=patience,monitor='val_loss')\n    rlop = ReduceLROnPlateau(patience=5)   \n    tb = TensorBoard(log_dir='{}{}'.format(tb_base_logdir,description))\n    cb = [cp,es,tb,rlop]\n    return cb'''\n\nhistory = emb_cont_model.fit(get_input(X_train_val), y_train_val, epochs=5, batch_size=16, \n          validation_data=(get_input(X_val), y_val)) #callbacks=set_callbacks()\n\nemb_cont_pred_test = emb_cont_model.predict(get_input(X_test))\nemb_cont_pred_train = emb_cont_model.predict(get_input(X_train_val))\n\nprint('test RMSE', mean_squared_error(y_test, emb_cont_pred_test) ** 0.5)\nprint('train RMSE', mean_squared_error(y_train_val, emb_cont_pred_train) ** 0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d3443252f8e67456d2c7c3d9c8c28487d34243d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}