{"cells":[{"metadata":{"_uuid":"1d0773bd33368b0c647c046609fdad9e9d422015"},"cell_type":"markdown","source":"I finally got around to building an autoencoder and thought that everyone would like to know what I was able to discover about the data.\n\n1. I made pivots on the data\n2. I built an autencoder\n3. I tracked the reconstruction accuracy pivots"},{"metadata":{"trusted":true,"_uuid":"3539540fcd3b8d23fa1d47ce6285b9c50ab32817"},"cell_type":"code","source":"from __future__ import print_function\n#from hdfs import InsecureClient\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold, RandomizedSearchCV, train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport keras\nfrom keras import regularizers\nfrom keras.layers import Input,Dropout,BatchNormalization,Activation,Add,PReLU, LSTM\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Dropout, Activation, Reshape, Flatten, Conv2D, MaxPooling2D\nfrom keras.optimizers import SGD\nfrom keras import backend as K\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard\nimport tensorflow as tf\n#import horovod.keras as hvd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75a07af253a0cdb8516912ae2522297df539b21e"},"cell_type":"code","source":"# Horovod: initialize Horovod.\n#hvd.init()\n\n# Horovod: pin GPU to be used to process local rank (one GPU per process)\n#config = tf.ConfigProto()\n#config.gpu_options.allow_growth = True\n#config.gpu_options.visible_device_list = str(hvd.local_rank())\n#K.set_session(tf.Session(config=config))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ceb7c8db0410f7f68619edc723e0f1652ff3437"},"cell_type":"markdown","source":"Below you can see the title of the pivot and the accuracy from reconstruction"},{"metadata":{"trusted":true,"_uuid":"db282a4a0d742a1b07de792f903b414b3750a40e"},"cell_type":"code","source":"#datasets with Reconstruction Accuracy\n#new_merchant_saleslag3_pivot - 10%\n#new_merchant_subsector_pivot_time - 40%\n#new_merchant_state_pivot_time - 0%\n#new_merchant_monthlag_pivot - 0%\n#new_merchant_id_pivot_time - 72%\n#new_card_id_state_pivot - 2%\n#new_card_id_monthlag_pivot - 13%\n#new_card_id_installment_pivot - 4%\n#new_card_id_feat3_pivot - 55%\n#new_card_id_feat2_pivot - 55%\n#new_card_id_feat1_pivot - 84%\n#new_card_id_cat3_pivot - 43%\n#new_card_id_cat2_pivot - 59%\n#new_card_id_cat1_pivot - 99%\n\n\n#hist_merchant_subsector_pivot_time - 65%\n#hist_merchant_state_pivot_time - 05%\n#hist_merchant_saleslag3_pivot - 75%\n#hist_merchant_monthlag_pivot_score - 95%\n#hist_merchant_monthlag_pivot - 57%\n#hist_merchant_id_pivot_time - 92%\n#hist_merchant_state_pivot_time - 3%\n#hist_merchant_saleslag3_pivot - 75%\n#hist_merchant_monthlag_pivot_score - 94%\n#hist_merchant_monthlag_pivot - 33%\n#hist_month_lag_real_price - 11%","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0cc8b926a174b7b12a5240a798943cbce2ed03a"},"cell_type":"code","source":"# load csv\ndf_train = pd.read_csv('../input/elo-merchant-category-recommendation/train.csv', index_col=['card_id'])\ndf_test = pd.read_csv('../input/elo-merchant-category-recommendation/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e73aa0df57a2982c7e19b7f6480c4f71798107b"},"cell_type":"code","source":"####find outliers\ndf_train['outliers'] = 0\ndf_train.loc[(df_train['target'] < -2) | (df_train['target'] > 2), 'outliers'] = 1\ntarget = df_train['target']\ndf_train['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b29702b99bd0b6f286abddaf0bca722c68bd9b5a"},"cell_type":"code","source":"df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target']]\ndf_test_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\n\ntarget = df_train['target']\n\ntrain_x, test_x = train_test_split(df_train[df_train_columns], test_size=0.2, random_state=420)\ntrain_x = train_x[train_x.outliers == 0] #where normal transactions\ntrain_x = train_x.drop(['outliers'], axis=1) #drop the class column\n\ntest_y = test_x['outliers'] #save the class column for the test set\ntest_x = test_x.drop(['outliers'], axis=1) #drop the class column\n\ntrain_x = train_x.values #transform to ndarray\ntest_x = test_x.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11a442c50394c2b19a171099fdf8c6c26bd37a06"},"cell_type":"code","source":"nb_epoch = 3\nbatch_size = 160\ninput_dim = train_x.shape[1] #num of columns, 30\nencoding_dim = 32\nhidden_dim = int(encoding_dim / 2) #i.e. 7\nlearning_rate = 0.001\n\ninput_layer = Input(shape=(input_dim, ))\nencoder = Dense(encoding_dim, activation=\"tanh\", activity_regularizer=regularizers.l1(learning_rate))(input_layer)\nencoder = Dense(hidden_dim, activation=\"relu\")(encoder)\ndecoder = Dense(hidden_dim, activation='tanh')(encoder)\ndecoder = Dense(input_dim, activation='relu')(decoder)\nautoencoder = Model(inputs=input_layer, outputs=decoder)\nautoencoder.summary()\n\n\nsgd = SGD(lr=0.001, momentum=0.02)\nautoencoder.compile(metrics=['accuracy'],\n                    loss='mean_squared_error',\n                    optimizer='sgd')\n\ncp = ModelCheckpoint(filepath=\"autoencoder_fraud.h5\",\n                               save_best_only=True,\n                               verbose=0)\n\ntb = TensorBoard(log_dir='./logs',\n                histogram_freq=0,\n                write_graph=True,\n                write_images=True)\n\n\n# Horovod: adjust learning rate based on number of GPUs.\n#opt = keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0, nesterov=True)\n\n# Horovod: add Horovod Distributed Optimizer.\n#opt = hvd.DistributedOptimizer(opt)\n\nhistory = autoencoder.fit(train_x, train_x,\n                    epochs=nb_epoch,\n                    batch_size=batch_size,\n                    shuffle=True,\n                    validation_data=(test_x, test_x),\n                    verbose=1,\n                    callbacks=[cp, tb]).history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6454e422986959ea0815fa8f49a2ceaa52a0062"},"cell_type":"code","source":"test_x_predictions = autoencoder.predict(test_x)\nmse = np.mean(np.power(test_x - test_x_predictions, 2), axis=1)\nerror_df = pd.DataFrame({'Reconstruction_error': mse,\n                        'True_class': test_y})\nerror_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85438876ded9add95e87bbd8391a867ac3206c54"},"cell_type":"code","source":"#error_df.sort_values(by='Reconstruction_error', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a73c0e3de1a012b31f0cda4f82b8b7c65e39ccad"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nLABELS = [\"Normal\",\"Unusual\"]\nthreshold_fixed = 0.3524\npred_y = [1 if e > threshold_fixed else 0 for e in error_df.Reconstruction_error.values]\nconf_matrix = confusion_matrix(error_df.True_class, pred_y)\n\nplt.figure(figsize=(12, 12))\nsns.heatmap(conf_matrix, xticklabels=LABELS, yticklabels=LABELS, annot=True, fmt=\"d\");\nplt.title(\"Confusion matrix\")\nplt.ylabel('True class')\nplt.xlabel('Predicted class')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6aca724cd2a205155a17ac976f31ab1546e2507"},"cell_type":"code","source":"mse=autoencoder.predict(df_train[df_test_columns])\ndf_train['anomoly'] = np.mean(np.power(df_train[df_test_columns] - mse, 2), axis=1)\ndf_train.anomoly.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f4d5f2cc23c1738a3ead2f1a29bcc182f4901a1"},"cell_type":"code","source":"mse=autoencoder.predict(df_test[df_test_columns])\ndf_test['anomoly'] = np.mean(np.power(df_test[df_test_columns] - mse, 2), axis=1)\ndf_test.anomoly.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"285d9f379ee213937b522c8e1b5674997aa6b6da"},"cell_type":"code","source":"ae_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"a5ea0479e21b2b8c6f44f8d3d4726334f243136e"},"cell_type":"code","source":"param = {'boosting': 'goss',\n         'objective': 'regression',\n         'metric': 'rmse',\n         'learning_rate': 0.01,\n         'subsample': 0.9855232997390695,\n         'max_depth': 7,\n         'top_rate': 0.9064148448434349,\n         'num_leaves': 63,\n         'min_child_weight': 41.9612869171337,\n         'other_rate': 0.0721768246018207,\n         'reg_alpha': 9.677537745007898,\n         'colsample_bytree': 0.5665320670155495,\n         'min_split_gain': 9.820197773625843,\n         'reg_lambda': 8.2532317400459,\n         'min_data_in_leaf': 21,\n         \"verbosity\": -1,\n#         \"device\" : \"gpu\",\n         \"n_jobs\" : -1}\n\nfolds = StratifiedKFold(n_splits=4, shuffle=True, random_state=4590)\noof = np.zeros(len(df_train))\npredictions = np.zeros(len(df_test))\nfeature_importance_df = pd.DataFrame()\nlabel_cols = [\"outliers\"]\ny_split = df_train[label_cols].values\n\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(y_split[:,0], y_split[:,0])):\n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(df_train.iloc[trn_idx][ae_columns], label=target.iloc[trn_idx])#, categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(df_train.iloc[val_idx][ae_columns], label=target.iloc[val_idx])#, categorical_feature=categorical_feats)\n\n    num_round = 5000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 50)\n    oof[val_idx] = clf.predict(df_train.iloc[val_idx][ae_columns], num_iteration=clf.best_iteration)\n\n    \n    train_prediction = clf.predict(df_train[ae_columns] / folds.n_splits)\n    predictions += clf.predict(df_test[ae_columns] / folds.n_splits)\n\nnp.sqrt(mean_squared_error(oof, target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0560dd01e0e71e37c2c5e12a1fd645334ba44414","_kg_hide-output":true,"_kg_hide-input":false},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":df_test[\"card_id\"].values})\nsub_df[\"target\"] = predictions\nsub_df.to_csv(\"submission_new.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}