{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"141761af838c246ec050c9e26833c90e02fc5aff"},"cell_type":"code","source":"#an attempt to use embedding for tabular data\n# thanks to https://www.kaggle.com/artgor/nn-baseline, https://www.kaggle.com/hrmello/starter-neural-network-3-939-lb\n# and kaggle kernel on feature engineering like https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-elo and others","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras_tqdm import TQDMNotebookCallback\n\nimport os\n\nimport tensorflow as tf\nimport keras as K\nprint(tf.__version__)\nprint(K.__version__)\nprint(tf.keras.__version__)\n\nSEED = 2018\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"140aa9604902a386ea9728cf032b5b6c6945ca83"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\n\nfrom sklearn import preprocessing\nimport warnings\nimport datetime\nwarnings.filterwarnings(\"ignore\")\nimport gc\n\nfrom scipy.stats import describe\n%matplotlib inline\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f2e00feb2eb574589e3e30b168b9037edcf809c"},"cell_type":"code","source":"#Loading Train and Test Data\ntrain = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])\nprint(\"{} observations and {} features in train set.\".format(train.shape[0],train.shape[1]))\nprint(\"{} observations and {} features in test set.\".format(test.shape[0],test.shape[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aec9df3fde4f11c69aeb81b6874836cc12734856"},"cell_type":"code","source":"train[\"month\"] = train[\"first_active_month\"].dt.month\ntest[\"month\"] = test[\"first_active_month\"].dt.month\ntrain[\"year\"] = train[\"first_active_month\"].dt.year\ntest[\"year\"] = test[\"first_active_month\"].dt.year\ntrain['elapsed_time'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\ntest['elapsed_time'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ae58580769cc2918430c169bcbe6b0f4a6fbdd4"},"cell_type":"code","source":"hist_trans = pd.read_csv(\"../input/historical_transactions.csv\")\nhist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"737090758d4ce0fe88fd1120a7957357965a5cac"},"cell_type":"code","source":"hist_trans = pd.get_dummies(hist_trans, columns=['category_2', 'category_3'])\nhist_trans['authorized_flag'] = hist_trans['authorized_flag'].map({'Y': 1, 'N': 0})\nhist_trans['category_1'] = hist_trans['category_1'].map({'Y': 1, 'N': 0})\nhist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"107c6305de28ae4ac118da28cfaa7d87d147c42c"},"cell_type":"code","source":"def aggregate_transactions(trans, prefix):  \n    trans.loc[:, 'purchase_date'] = pd.DatetimeIndex(trans['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['mean'],\n        'category_2_1.0': ['mean'],\n        'category_2_2.0': ['mean'],\n        'category_2_3.0': ['mean'],\n        'category_2_4.0': ['mean'],\n        'category_2_5.0': ['mean'],\n        'category_3_A': ['mean'],\n        'category_3_B': ['mean'],\n        'category_3_C': ['mean'],\n        'merchant_id': ['nunique'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max']\n    }\n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e0ec7191a3b01e1f938a4f2c16125e2e0bf8a1f"},"cell_type":"code","source":"import gc\nmerch_hist = aggregate_transactions(hist_trans, prefix='hist_')\ndel hist_trans\ngc.collect()\ntrain = pd.merge(train, merch_hist, on='card_id',how='left')\ntest = pd.merge(test, merch_hist, on='card_id',how='left')\ndel merch_hist\ngc.collect()\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4f0316fc25f33ae5a9974ac227f21303b518c3e"},"cell_type":"code","source":"new_trans = pd.read_csv(\"../input/new_merchant_transactions.csv\")\nnew_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd7eb65f0f80784a75c86eca8925c986a8458ef"},"cell_type":"code","source":"new_trans = pd.get_dummies(new_trans, columns=['category_2', 'category_3'])\nnew_trans['authorized_flag'] = new_trans['authorized_flag'].map({'Y': 1, 'N': 0})\nnew_trans['category_1'] = new_trans['category_1'].map({'Y': 1, 'N': 0})\nnew_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"923db5a03f55769f964674ecfd7186b6abc250a5"},"cell_type":"code","source":"merch_new = aggregate_transactions(new_trans, prefix='new_')\ndel new_trans\ngc.collect()\ntrain = pd.merge(train, merch_new, on='card_id',how='left')\ntest = pd.merge(test, merch_new, on='card_id',how='left')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b013b96b481cd58ae2576efa88f968e16f6cac1b"},"cell_type":"code","source":"del merch_new\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04e8c8d115e6216db7fa294315d7c06ee157a95c"},"cell_type":"code","source":"cat_cols = ['feature_1', 'feature_2', 'feature_3', 'month', 'year','hist_merchant_id_nunique']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c21b72104f0a1f3b5c1860dbf62fe51ae773b3ff"},"cell_type":"code","source":"target = train['target']\ndrops = ['card_id', 'first_active_month', 'target']\nnum_cols = [col for col in train.columns if col not in cat_cols and col not in drops]\ntotal_cols = [col for col in train.columns]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75dab85b0df9fbf20d5a46dcd328c77917c34e2a"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nmax_values = {}\nfor col in cat_cols:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))\n    max_values[col] = max(train[col].max(), test[col].max())  + 2\n    print(max_values[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e8fb49b5420bd91c5ede46b543e073e5c1c7d8a"},"cell_type":"code","source":"# printing because I'm too lazy to write everything by hand. Open output to see.\nfor col in cat_cols:\n    n = col.replace('.', '_')\n    print(f'{n} = Input(shape=[1], name=\"{col}\")')\n    print(f'emb_{n} = Embedding({max_values[col]}, {(np.min(max_values[col]+1)//2, 50)})({col})')\n    print(',', n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5efae9f6ec6e3ca86d5bdad9fdfa6e423e313d78"},"cell_type":"code","source":"#filter num values\ntarget = train['target']\ndrops = ['card_id', 'first_active_month', 'target']\nuse_cols_num = [c for c in train.columns if c in num_cols]\nfeatures_num = list(train[use_cols_num].columns)\ntrain[features_num].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abaafe60c37114cb800dd54c58b8771e8afcc50f"},"cell_type":"code","source":"#scale the data and impute the null values \n#note: apparently, GPU environment doesn't have an updated version of sklearn,\n#so we cannot use sklearn.impute.SimpleImputer. In CPU environement this is possible\nfrom sklearn.preprocessing import StandardScaler, Imputer\nsc = StandardScaler()\ntrain[features_num] = train[features_num].fillna(0)\ntrain[features_num] = sc.fit_transform(train[features_num])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"664212fb33470d36421b540a6276c612f1d655e4"},"cell_type":"code","source":"#fit test data\ntest[features_num] = test[features_num].fillna(0)\ntest[features_num] = sc.fit_transform(test[features_num])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"583b8181297c4a5f0bac6113f4964ea42ea7cf01"},"cell_type":"code","source":"#drop columns\nX_train = train.drop([col for col in drops if col in train.columns], axis=1)\nX_test = test.drop([col for col in drops if col in test.columns], axis=1)\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfd5fb4668963bb01867e4db0c40099bb2fada64","scrolled":true},"cell_type":"code","source":"#renommer certaines colonnes\nX_train.rename(columns={'hist_category_2_1.0_mean': 'hist_category_2_1_0_mean',\n                        'hist_category_2_2.0_mean': 'hist_category_2_2_0_mean',\n                        'hist_category_2_3.0_mean': 'hist_category_2_3_0_mean',\n                        'hist_category_2_4.0_mean': 'hist_category_2_4_0_mean',\n                        'hist_category_2_5.0_mean': 'hist_category_2_5_0_mean',\n                        'new_category_2_1.0_mean': 'new_category_2_1_0_mean',\n                        'new_category_2_2.0_mean': 'new_category_2_2_0_mean',\n                        'new_category_2_3.0_mean': 'new_category_2_3_0_mean',\n                        'new_category_2_4.0_mean': 'new_category_2_4_0_mean',\n                        'new_category_2_5.0_mean': 'new_category_2_5_0_mean'}, inplace=True)\ntotal_cols = [col for col in X_train.columns]\ntotal_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7ce9089c7f947a96d3cf4b6287475f181ff642d"},"cell_type":"code","source":"#renommer certaines colonnes\nX_test.rename(columns={'hist_category_2_1.0_mean': 'hist_category_2_1_0_mean',\n                        'hist_category_2_2.0_mean': 'hist_category_2_2_0_mean',\n                        'hist_category_2_3.0_mean': 'hist_category_2_3_0_mean',\n                        'hist_category_2_4.0_mean': 'hist_category_2_4_0_mean',\n                        'hist_category_2_5.0_mean': 'hist_category_2_5_0_mean',\n                        'new_category_2_1.0_mean': 'new_category_2_1_0_mean',\n                        'new_category_2_2.0_mean': 'new_category_2_2_0_mean',\n                        'new_category_2_3.0_mean': 'new_category_2_3_0_mean',\n                        'new_category_2_4.0_mean': 'new_category_2_4_0_mean',\n                        'new_category_2_5.0_mean': 'new_category_2_5_0_mean'}, inplace=True)\ntotal_cols = [col for col in X_test.columns]\ntotal_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70bfea80d2dea43d5ebd3ec003a59a669dedd24a"},"cell_type":"code","source":"num_cols = [col for col in X_train.columns if col not in cat_cols and col not in drops]\nnum_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3aef0c615c09489a3ff20f8fced9b60712cf5e7"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.layers import Dense, BatchNormalization, Dropout, Input\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom keras.models import Sequential\nfrom sklearn.model_selection import train_test_split\ntrain_y = target.values\nx_train, x_val, y_train, y_val = train_test_split(X_train, train_y, test_size = .1, random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da94342a64a60859a05307a4d35362a121b9251d"},"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit, KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dropout, Dense, Embedding, SpatialDropout1D, concatenate, BatchNormalization, Flatten\n\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.losses import mean_squared_error as mse_loss\n\nfrom keras import optimizers\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n#building the network\n\nfrom keras.initializers import he_normal, he_uniform,  glorot_normal,  glorot_uniform\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"143dc2d6dfb2470f8261cb889e86982fb3a938e2"},"cell_type":"code","source":"def rmse(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e146b8a1108e89a0d6441a622ed16800880a03a"},"cell_type":"code","source":"def model(dense_dim_1=128, dense_dim_2=64, dense_dim_3=256, dense_dim_4=512,  dense_dim_5=512, dense_dim_6=256,\ndropout1=0.1, dropout2=0.1, dropout3=0.2, dropout4=0.2, dropout5=0.2, dropout6=0.2, lr=0.0001):\n\n    #Inputs cat\n    feature_1 = Input(shape=[1], name=\"feature_1\")\n    feature_2 = Input(shape=[1], name=\"feature_2\")\n    feature_3 = Input(shape=[1], name=\"feature_3\")\n    month = Input(shape=[1], name=\"month\")\n    year = Input(shape=[1], name=\"year\")\n    hist_merchant_id_nunique = Input(shape=[1], name=\"hist_merchant_id_nunique\")\n    \n    #Input num    \n    elapsed_time = Input(shape=[1], name=\"elapsed_time\")\n    hist_transactions_count = Input(shape=[1], name=\"hist_transactions_count\")\n    hist_authorized_flag_sum = Input(shape=[1], name=\"hist_authorized_flag_sum\")\n    hist_authorized_flag_mean = Input(shape=[1], name=\"hist_authorized_flag_mean\")\n    hist_category_1_mean = Input(shape=[1], name=\"hist_category_1_mean\")\n    hist_category_2_1_0_mean = Input(shape=[1], name=\"hist_category_2_1_0_mean\")\n    hist_category_2_2_0_mean = Input(shape=[1], name=\"hist_category_2_2_0_mean\")\n    hist_category_2_3_0_mean = Input(shape=[1], name=\"hist_category_2_3_0_mean\")\n    hist_category_2_4_0_mean = Input(shape=[1], name=\"hist_category_2_4_0_mean\")\n    hist_category_2_5_0_mean = Input(shape=[1], name=\"hist_category_2_5_0_mean\")\n    hist_category_3_A_mean = Input(shape=[1], name=\"hist_category_3_A_mean\")\n    hist_category_3_B_mean = Input(shape=[1], name=\"hist_category_3_B_mean\")\n    hist_category_3_C_mean = Input(shape=[1], name=\"hist_category_3_C_mean\")\n    hist_purchase_amount_sum = Input(shape=[1], name=\"hist_purchase_amount_sum\")\n    hist_purchase_amount_mean = Input(shape=[1], name=\"hist_purchase_amount_mean\")\n    hist_purchase_amount_max = Input(shape=[1], name=\"hist_purchase_amount_max\")\n    hist_purchase_amount_min = Input(shape=[1], name=\"hist_purchase_amount_min\")\n    hist_purchase_amount_std = Input(shape=[1], name=\"hist_purchase_amount_std\")\n    hist_installments_sum = Input(shape=[1], name=\"hist_installments_sum\")\n    hist_installments_mean = Input(shape=[1], name=\"hist_installments_mean\")\n    hist_installments_max = Input(shape=[1], name=\"hist_installments_max\")\n    hist_installments_min = Input(shape=[1], name=\"hist_installments_min\")\n    hist_installments_std = Input(shape=[1], name=\"hist_installments_std\")\n    hist_purchase_date_ptp = Input(shape=[1], name=\"hist_purchase_date_ptp\")\n    hist_month_lag_min = Input(shape=[1], name=\"hist_month_lag_min\")\n    hist_month_lag_max = Input(shape=[1], name=\"hist_month_lag_max\")\n    \n    new_transactions_count = Input(shape=[1], name=\"new_transactions_count\")\n    new_authorized_flag_sum = Input(shape=[1], name=\"new_authorized_flag_sum\")\n    new_authorized_flag_mean = Input(shape=[1], name=\"new_authorized_flag_mean\")\n    new_category_1_mean = Input(shape=[1], name=\"new_category_1_mean\")\n    new_category_2_1_0_mean = Input(shape=[1], name=\"new_category_2_1_0_mean\")\n    new_category_2_2_0_mean = Input(shape=[1], name=\"new_category_2_2_0_mean\")\n    new_category_2_3_0_mean = Input(shape=[1], name=\"new_category_2_3_0_mean\")\n    new_category_2_4_0_mean = Input(shape=[1], name=\"new_category_2_4_0_mean\")\n    new_category_2_5_0_mean = Input(shape=[1], name=\"new_category_2_5_0_mean\")\n    new_category_3_A_mean = Input(shape=[1], name=\"new_category_3_A_mean\")\n    new_category_3_B_mean = Input(shape=[1], name=\"new_category_3_B_mean\")\n    new_category_3_C_mean = Input(shape=[1], name=\"new_category_3_C_mean\")\n    new_purchase_amount_sum = Input(shape=[1], name=\"new_purchase_amount_sum\")\n    new_purchase_amount_mean = Input(shape=[1], name=\"new_purchase_amount_mean\")\n    new_purchase_amount_max = Input(shape=[1], name=\"new_purchase_amount_max\")\n    new_purchase_amount_min = Input(shape=[1], name=\"new_purchase_amount_min\")\n    new_purchase_amount_std = Input(shape=[1], name=\"new_purchase_amount_std\")\n    new_installments_sum = Input(shape=[1], name=\"new_installments_sum\")\n    new_installments_mean = Input(shape=[1], name=\"new_installments_mean\")\n    new_installments_max = Input(shape=[1], name=\"new_installments_max\")\n    new_installments_min = Input(shape=[1], name=\"new_installments_min\")\n    new_installments_std = Input(shape=[1], name=\"new_installments_std\")\n    new_purchase_date_ptp = Input(shape=[1], name=\"new_purchase_date_ptp\")\n    new_month_lag_min = Input(shape=[1], name=\"new_month_lag_min\")\n    new_month_lag_max = Input(shape=[1], name=\"new_month_lag_max\")    \n    \n    #Embeddings layers\n\n    emb_feature_1 = Embedding(6, 3)(feature_1)\n    emb_feature_2 = Embedding(4, 3)(feature_2)\n    emb_feature_3 = Embedding(4, 3)(feature_3)\n    emb_month = Embedding(26, 13)(month)\n    emb_year = Embedding(18, 9)(year)\n    emb_hist_merchant_id_nunique = Embedding(333, 50)(hist_merchant_id_nunique)\n\n\n    concat_emb1 = concatenate([\n           Flatten() (emb_feature_1),\n            Flatten() (emb_feature_2),\n            Flatten() (emb_feature_3),\n            Flatten() (emb_month),\n            Flatten() (emb_year),\n            Flatten() (emb_hist_merchant_id_nunique)\n    ])\n    \n    categ = Dropout(dropout1)(Dense(dense_dim_1,kernel_initializer=he_uniform(seed=SEED),activation='relu') (concat_emb1))\n    categ = BatchNormalization()(categ)\n    categ = Dropout(dropout2)(Dense(dense_dim_2,kernel_initializer=he_uniform(seed=SEED),activation='relu') (categ))\n        \n    #main layer\n    main_l = concatenate([\n          categ\n        , elapsed_time\n        , hist_transactions_count\n        , hist_authorized_flag_sum\n        , hist_authorized_flag_mean\n        , hist_category_1_mean\n        , hist_category_2_1_0_mean\n        , hist_category_2_2_0_mean\n        , hist_category_2_3_0_mean\n        , hist_category_2_4_0_mean \n        , hist_category_2_5_0_mean\n        , hist_category_3_A_mean\n        , hist_category_3_B_mean\n        , hist_category_3_C_mean\n        , hist_purchase_amount_sum\n        , hist_purchase_amount_mean\n        , hist_purchase_amount_max\n        , hist_purchase_amount_min\n        , hist_purchase_amount_std\n        , hist_installments_sum\n        , hist_installments_mean\n        , hist_installments_max\n        , hist_installments_min\n        , hist_installments_std \n        , hist_purchase_date_ptp\n        , hist_month_lag_min\n        , hist_month_lag_max\n        , new_transactions_count\n        , new_authorized_flag_sum\n        , new_authorized_flag_mean\n        , new_category_1_mean\n        , new_category_2_1_0_mean\n        , new_category_2_2_0_mean\n        , new_category_2_3_0_mean\n        , new_category_2_4_0_mean \n        , new_category_2_5_0_mean\n        , new_category_3_A_mean\n        , new_category_3_B_mean\n        , new_category_3_C_mean\n        , new_purchase_amount_sum\n        , new_purchase_amount_mean\n        , new_purchase_amount_max\n        , new_purchase_amount_min\n        , new_purchase_amount_std\n        , new_installments_sum\n        , new_installments_mean\n        , new_installments_max\n        , new_installments_min\n        , new_installments_std \n        , new_purchase_date_ptp\n        , new_month_lag_min\n        , new_month_lag_max\n    ])\n    \n    main_l = Dropout(dropout3)(Dense(dense_dim_3,kernel_initializer=he_uniform(seed=SEED),activation='relu') (main_l))\n    main_l = BatchNormalization()(main_l)\n    main_l = Dropout(dropout4)(Dense(dense_dim_4,kernel_initializer=he_uniform(seed=SEED),activation='relu') (main_l))\n    main_l = BatchNormalization()(main_l)\n    main_l = Dropout(dropout5)(Dense(dense_dim_5,kernel_initializer=he_uniform(seed=SEED),activation='relu') (main_l))\n    main_l = BatchNormalization()(main_l) \n    main_l = Dropout(dropout6)(Dense(dense_dim_6,kernel_initializer=he_uniform(seed=SEED),activation='relu') (main_l))\n\n    \n    #output\n    output = Dense(1) (main_l)\n\n    model = Model([feature_1,\n                    feature_2,\n                    feature_3,\n                    month,\n                    year,\n                    hist_merchant_id_nunique,\n                    elapsed_time,\n                    hist_transactions_count,\n                    hist_authorized_flag_sum,\n                    hist_authorized_flag_mean,\n                    hist_category_1_mean,\n                    hist_category_2_1_0_mean,\n                    hist_category_2_2_0_mean,\n                    hist_category_2_3_0_mean,\n                    hist_category_2_4_0_mean, \n                    hist_category_2_5_0_mean,\n                    hist_category_3_A_mean,\n                    hist_category_3_B_mean,\n                    hist_category_3_C_mean,\n                    hist_purchase_amount_sum,\n                    hist_purchase_amount_mean,\n                    hist_purchase_amount_max,\n                    hist_purchase_amount_min,\n                    hist_purchase_amount_std,\n                    hist_installments_sum,\n                    hist_installments_mean,\n                    hist_installments_max,\n                    hist_installments_min,\n                    hist_installments_std, \n                    hist_purchase_date_ptp,\n                    hist_month_lag_min,\n                    hist_month_lag_max,\n                    new_transactions_count,\n                    new_authorized_flag_sum,\n                    new_authorized_flag_mean,\n                    new_category_1_mean,\n                    new_category_2_1_0_mean,\n                    new_category_2_2_0_mean,\n                    new_category_2_3_0_mean,\n                    new_category_2_4_0_mean, \n                    new_category_2_5_0_mean,\n                    new_category_3_A_mean,\n                    new_category_3_B_mean,\n                    new_category_3_C_mean,\n                    new_purchase_amount_sum,\n                    new_purchase_amount_mean,\n                    new_purchase_amount_max,\n                    new_purchase_amount_min,\n                    new_purchase_amount_std,\n                    new_installments_sum,\n                    new_installments_mean,\n                    new_installments_max,\n                    new_installments_min,\n                    new_installments_std, \n                    new_purchase_date_ptp,\n                    new_month_lag_min,\n                    new_month_lag_max], output)\n\n    #model = Model([**params], output)\n    model.compile(optimizer = Adam(lr=lr),\n                  loss= rmse,\n                  metrics=[rmse])\n    return model\n\n\n#model=model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f14633501b711027976493ee08a6ffe4d1bdf18"},"cell_type":"code","source":"# converting data to format which can be used by Keras\ndef get_keras_data(df, num_cols, cat_cols):\n    cols = num_cols + cat_cols\n\n    X = {col: np.array(df[col]) for col in cols}\n    # print(\"Data ready for Vectorization\")\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9d74c7322710dcea9e491328d01de4f918e0e20"},"cell_type":"code","source":"X_test_keras = get_keras_data(X_test, num_cols, cat_cols)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d10a38cae4bf9627add6b1574f7f6eb424d14821"},"cell_type":"code","source":"def train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, reduce_lr=True, patience=7):\n    \"\"\"\n    Helper function to train model. Also I noticed that ReduceLROnPlateau is rarely\n    useful, so added an option to turn it off.\n    \"\"\"\n    \n    early_stopping = EarlyStopping(patience=patience, verbose=1)\n    model_checkpoint = ModelCheckpoint(\"model.hdf5\",\n                                       save_best_only=True, verbose=1, monitor='val_loss', mode='min')\n    if reduce_lr:\n        reduce_lr = ReduceLROnPlateau(factor=0.1, patience=2, min_lr=0.000005, verbose=1)\n        hist = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,\n                            validation_data=(X_v, y_valid), verbose=True,\n                            callbacks=[early_stopping, model_checkpoint, reduce_lr])\n    \n    else:\n        hist = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,\n                            validation_data=(X_v, y_valid), verbose=True,\n                            callbacks=[early_stopping, model_checkpoint])\n\n    \n    return hist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0556c2eea86be7ef1195eec594d12b33f936fe32"},"cell_type":"code","source":"X_t = get_keras_data(x_train, num_cols, cat_cols)\nX_v = get_keras_data(x_val, num_cols, cat_cols)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c1c6c07e935ceca7be7b2391ed5ddfe2db29646"},"cell_type":"code","source":"keras_model = model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"474aa6733dfe3d87de096ce00972dd60df93e614"},"cell_type":"code","source":"keras_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"832e589381dcf4b5950e3a2b7ed8179cb60195ad"},"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(keras_model, to_file='model.png')\n\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(keras_model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab555ac9c73c9eeb4beaeab223e065da60aceee9","scrolled":true},"cell_type":"code","source":"batch_size = 64\nepochs = 100\nhist = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_val, reduce_lr=True, patience=7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83498c11c23b65e6925231408e6137ea8f645a8a"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n#plotting training and validations losses\nplt.plot(hist.history['val_loss'], label = \"val_loss\")\nplt.plot(hist.history['loss'], label = \"loss\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fe56deb7ca7e80285b9b0c7fd53249140abb505"},"cell_type":"code","source":"keras_modelmax = load_model(\"model.hdf5\", custom_objects={'rmse': rmse})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"450d984ac60faa443603f9f2608e1df41d85f338"},"cell_type":"code","source":"y_pred_valid = keras_modelmax.predict(X_v)\nvalid_score = mean_squared_error(y_val,y_pred_valid)** 0.5\nprint(\"Validation score: \", valid_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18aa9dc289e25f1258dd442a47da2aa7dd9b9eac"},"cell_type":"code","source":"predictions = keras_modelmax.predict(X_test_keras).reshape(-1,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2c2995a4d107a8c2ea770123e87665bd7a7b7f9"},"cell_type":"code","source":"#saving the card_ids\nids = test['card_id'].values\nsubmission = pd.DataFrame(ids, columns=['card_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb47e8810a71b294fd64b98845478d75725b2552"},"cell_type":"code","source":"submission['target'] = predictions\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"214b4d740c03981f5039413c6e55a5ab122ccc34"},"cell_type":"code","source":"submission.to_csv(\"submission_neuralnet.csv\", index = False, header = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e1c8ce8aad46711db93d999ce5a90f155243b44"},"cell_type":"code","source":"print(\"done\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}