{"cells":[{"metadata":{"_uuid":"be9fb52556e8a31b451c9e8d3b102faf4e445d8e"},"cell_type":"markdown","source":"Here is a basic guide to use if someone wants to use neural networks in this competition. Although LGB is by far the most used algorithm for this kind of competition in Kaggle, it is fun to try different routes and see where they lead.\n\nNote: the feature engineering steps were borrowed from [SRK's kernel](https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-elo)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.layers import Dense, BatchNormalization, Dropout, Input\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom keras.models import Sequential\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, Imputer\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"#reading the files\ndata_dir = \"../input/\"\nmerchants = pd.read_csv(os.path.join(data_dir, \"merchants.csv\"))\nhistorical = pd.read_csv(os.path.join(data_dir, \"historical_transactions.csv\"), )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e61cb2c57e0b17a4e966db796fa3cc9963f7ff7"},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(data_dir, \"train.csv\"),  parse_dates=[\"first_active_month\"])\ntest_df = pd.read_csv(os.path.join(data_dir, \"test.csv\"),  parse_dates=[\"first_active_month\"])\nnew_trans_df = pd.read_csv(os.path.join(data_dir, \"new_merchant_transactions.csv\"))\n\ngdf = historical.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].agg(['sum', 'mean', 'std', 'min', 'max']).reset_index()\ngdf.columns = [\"card_id\", \"sum_hist_trans\", \"mean_hist_trans\", \"std_hist_trans\", \"min_hist_trans\", \"max_hist_trans\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")\n\ngdf = new_trans_df.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].size().reset_index()\ngdf.columns = [\"card_id\", \"num_merch_transactions\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5261534e0782c51af3ed278517eb20ce30365e1f"},"cell_type":"code","source":"gdf = new_trans_df.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].agg(['sum', 'mean', 'std', 'min', 'max']).reset_index()\ngdf.columns = [\"card_id\", \"sum_merch_trans\", \"mean_merch_trans\", \"std_merch_trans\", \"min_merch_trans\", \"max_merch_trans\"]\ntrain_df = pd.merge(train_df, gdf, on=\"card_id\", how=\"left\")\ntest_df = pd.merge(test_df, gdf, on=\"card_id\", how=\"left\")\n\ntrain_df[\"year\"] = train_df[\"first_active_month\"].dt.year\ntest_df[\"year\"] = test_df[\"first_active_month\"].dt.year\ntrain_df[\"month\"] = train_df[\"first_active_month\"].dt.month\ntest_df[\"month\"] = test_df[\"first_active_month\"].dt.month\n\ncols_to_use = [\"feature_1\", \"feature_2\", \"feature_3\", \n               \"sum_hist_trans\", \"mean_hist_trans\", \"std_hist_trans\", \n               \"min_hist_trans\", \"max_hist_trans\",\n               \"year\", \"month\",\"num_merch_transactions\", \n                \"sum_merch_trans\", \"mean_merch_trans\", \"std_merch_trans\",\n                \"min_merch_trans\", \"max_merch_trans\",\n              ]\n\n#get train and test dataframes\ntrain_X = train_df[cols_to_use]\ntest_X = test_df[cols_to_use]\ntrain_y = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96582e463530b69290d5ba938dc3d552f0d3518e","scrolled":true},"cell_type":"code","source":"#scale the data and impute the null values \n#note: apparently, GPU environment doesn't have an updated version of sklearn,\n#so we cannot use sklearn.impute.SimpleImputer. In CPU environement this is possible\nsc = StandardScaler()\ntrain_X = train_X.fillna(0)\ntrain_X = sc.fit_transform(train_X)\nx_train, x_val, y_train, y_val = train_test_split(train_X, train_y, test_size = .1, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"fa1ed3d513a957b36a6e4ce8452e33442fd8fc43"},"cell_type":"code","source":"#building the network\nimport keras.backend as K\n#definind the rmse metric\ndef rmse(y_true, y_pred):\n        return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))\n\nfh_neurons = 1024 #first hidden layer\ndrop_rate = 0.7\n\n#the model is just a sequence of fully connected layers, batch normalization and dropout using ELUs as activation functions\nmodel = Sequential()\nmodel.add(Dense(fh_neurons, input_dim=x_train.shape[1], activation='elu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(drop_rate))\nmodel.add(Dense(fh_neurons*2, activation='elu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(drop_rate))\nmodel.add(Dense(fh_neurons*2, activation='elu'))\nmodel.add(Dense(fh_neurons, activation='elu'))\nmodel.add(Dense(1, activation='linear'))\n\nmodel.compile(optimizer='adam',loss=rmse)\nearly_stopping = EarlyStopping(monitor = 'val_loss', patience = 5)\ncheckpointer = ModelCheckpoint(filepath='weights.hdf5', \n                               verbose=1, save_best_only=True)\n\nmodel.fit(x_train, y_train, validation_data = (x_val, y_val), epochs=15, batch_size=256, callbacks = [early_stopping, checkpointer])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29a0179b05abedc422c218a431090d5cdb13272d"},"cell_type":"markdown","source":"One issue that I find here is that both training and validation loss values are way lower than the LB values, which seems to indicate that this particular model is overfitting the training data, but since the validation loss is in agreement with the training loss, I'm a bit confused about it. If anyone knows what may be going on, feel free to comment :)"},{"metadata":{"trusted":true,"_uuid":"0ca04304b82902e0c4a96199f4557bc42c0d4e88"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n#plotting training and validations losses\nplt.plot(model.history.history['val_loss'], label = \"val_loss\")\nplt.plot(model.history.history['loss'], label = \"loss\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53c26cd88b6d146fc3ee16efa7b0bda64bd2280f"},"cell_type":"code","source":"#saving the card_ids\nids = test_df['card_id'].values\nsubmission = pd.DataFrame(ids, columns=['card_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4eb8bef0c2c632eefc357a806da5f2ac294c1163"},"cell_type":"code","source":"#making the predictions\ntest_df = test_df[cols_to_use]\ntest_df = test_df.fillna(0)\ntest_df = sc.transform(test_df)\npredictions = model.predict(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e09a7c2aace883642e5d341059f4c9e243736e2"},"cell_type":"code","source":"submission['target'] = predictions.flatten()\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f691d10343a948d7c42ef2e1fc0b541055607a2a"},"cell_type":"code","source":"submission.to_csv(\"submission_neuralnet.csv\", index = False, header = True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}