{"cells":[{"metadata":{},"cell_type":"markdown","source":"I followed the [training](https://www.tensorflow.org/tutorials/structured_data/imbalanced_data) offered by TensorFlow on uing NN on unbalanced data for this work.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Import required libraries\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading training data\nraw_train_df = pd.read_csv('/kaggle/input/fi-2020-q2-kaggle-competition/train.csv')\n\n# Loading data for prediction\nraw_predict_df = pd.read_csv('/kaggle/input/fi-2020-q2-kaggle-competition/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Taking a copy of the imported data\ncleaned_train_df = raw_train_df.copy()\n\n# Adding 0.01 to amounts to avoid zeo values and taking log\ncleaned_train_df['Amount'] = np.log(cleaned_train_df['Amount'] + 0.01)\n\n# Same proecdure on data for prediction\ncleaned_test_df = raw_predict_df.copy()\ncleaned_test_df['Amount'] = np.log(cleaned_test_df['Amount'] + 0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Splitting data for training, validation and testing\ntrain_df, test_df = train_test_split(cleaned_train_df, test_size=0.2)\ntrain_df, validation_df = train_test_split(train_df, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating array of labels\ntrain_labels = np.array(train_df['Class'])\n\n# A logical array of training labels\ntrain_labels_bool = train_labels != 0\n\n# Labels for validation\nvalidation_labels = np.array(validation_df['Class'])\n\n# Labels for test\ntest_labels = np.array(test_df['Class'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extracting features\ntrain_features = np.array(train_df[train_df.columns[1:-1]])\nvalidation_features = np.array(validation_df[validation_df.columns[1:-1]])\ntest_features = np.array(test_df[test_df.columns[1:-1]])\n\n# Data structure for prediction data is different\npredict_features = np.array(cleaned_test_df[cleaned_test_df.columns[2:]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Standardizing numeric features\nscaler_obj = StandardScaler()\ntrain_features = scaler_obj.fit_transform(train_features)\nvalidation_features = scaler_obj.fit_transform(validation_features)\ntest_features = scaler_obj.fit_transform(test_features)\npredict_features = scaler_obj.fit_transform(predict_features)\n\n# Checking range of training data\nmax([max(train_features[i]) for i in range(train_features.shape[1])])\nmin([min(train_features[i]) for i in range(train_features.shape[1])])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Limiting the range of features to [-5, 5]\ntrain_features = np.clip(train_features, -5, 5)\nvalidation_features = np.clip(validation_features, -5, 5)\ntest_features = np.clip(test_features, -5, 5)\npredict_features = np.clip(predict_features, -5, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper function that generate model\ndef model_gen(dense_nodes=16, output_bias=None):\n    METRICS = [\n        keras.metrics.TruePositives(name='tp'),\n        keras.metrics.TrueNegatives(name='tn'),\n        keras.metrics.FalsePositives(name='fp'),\n        keras.metrics.FalseNegatives(name='fn'),\n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall'),\n        keras.metrics.AUC(name='auc')\n    ]\n\n    if output_bias is not None:\n        output_bias = tf.keras.initializers.Constant(output_bias)\n\n    model = keras.Sequential([\n        keras.layers.Dense(\n            dense_nodes,\n            activation='relu',\n            input_shape=(train_features.shape[-1],)\n        ),\n        keras.layers.Dropout(0.5),\n        keras.layers.Dense(\n            1,\n            activation='sigmoid',\n            bias_initializer=output_bias\n        ),\n    ])\n\n    model.compile(\n        optimizer=keras.optimizers.Adam(lr=1e-3),\n        loss=keras.losses.BinaryCrossentropy(),\n        metrics=METRICS\n    )\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper for graphing model performance\ndef graph_model(model):\n    metrics = ['loss', 'auc', 'recall']\n    for i, metric in enumerate(metrics):\n        name = metric.replace(\"_\", \" \").capitalize()\n        plt.subplot(2,2, i+1)\n        plt.plot(\n            model.epoch,\n            model.history[metric],\n            label='Train'\n        )\n\n        plt.plot(\n            model.epoch,\n            model.history['val_'+metric],\n            linestyle=\"--\",\n            label='Val'\n        )\n        plt.xlabel('Epoch')\n        plt.ylabel(name)\n        plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot Confusion Matrix\ndef plot_cm(labels, predictions, p=0.5):\n    cm = confusion_matrix(labels, predictions > p)\n    plt.figure(figsize=(5, 5))\n    sns.heatmap(cm, annot=True, fmt='d')\n    plt.title(f\"Confusion Matrix @ %{p*100}\")\n    plt.xlabel(\"Predicted Label\")\n    plt.ylabel(\"Actual Label\")\n\n    print(\n        f\"True Negatives: {cm[0][0]}\\n\"\n        f\"False Negatives: {cm[1][0]}\\n\"\n        f\"True Positives: {cm[1][1]}\\n\"\n        f\"False Positives: {cm[0][1]}\"\n    )\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot ROC curve\ndef plot_roc(name, labels, predictions):\n    fp, tp, _ = sklearn.metrics.roc_curve(labels, predictions)\n\n    plt.plot(100*fp, 100*tp, label=name, linewidth=2)\n    plt.xlabel(\"False Positives (%)\")\n    plt.ylabel(\"True Positives (%)\")\n    plt.xlim([-0.5, 20])\n    plt.ylim([80, 100.5])\n    plt.grid(True)\n    ax = plt.gca()\n    ax.set_aspect('equal')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper function that return tf dataset\ndef make_ds(features, labels, buffer_size=100000):\n    ds = tf.data.Dataset.from_tensor_slices((features, labels))\n    ds = ds.shuffle(buffer_size).repeat()\n    return ds\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper function to over sample positive values, we do 50/50 on positive/negative records\ndef resample_ds():\n    pos_features = train_features[train_labels_bool]\n    neg_features = train_features[~train_labels_bool]\n\n    pos_labels = train_labels[train_labels_bool]\n    neg_labels = train_labels[~train_labels_bool]\n\n    pos_ds = make_ds(pos_features, pos_labels)\n    neg_ds = make_ds(neg_features, neg_labels)\n\n    resampled_ds = tf.data.experimental.sample_from_datasets(\n        [pos_ds, neg_ds],\n        weights=[0.5, 0.5]\n    )\n    resampled_ds = resampled_ds.batch(BATCH_SIZE).prefetch(1)\n    steps = np.ceil(len(neg_labels)/BATCH_SIZE)\n    return resampled_ds, steps","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Early stopper to avoid overfitting\nearly_stopping = tf.keras.callbacks.EarlyStopping(\n    monitor='val_auc',\n    verbose=1,\n    patience=10,\n    mode='max',\n    restore_best_weights=True\n)\n\n# Setting parameters\nBATCH_SIZE = 2048\nEPOCH = 100\nNODES = 24","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using helper function to resample training data\nresambled_ds, steps = resample_ds()\n\n# Creating the model\nresample_model = model_gen()\n\n# Creating datasets for validation\nval_ds = tf.data.Dataset.from_tensor_slices((validation_features, validation_labels)).cache()\nval_ds = val_ds.batch(BATCH_SIZE).prefetch(1)\n\n# Training the model\nresample_history = resample_model.fit(\n    resambled_ds,\n    epochs=EPOCH,\n    steps_per_epoch=steps,\n    callbacks=[early_stopping],\n    validation_data=val_ds\n)\n\n# Generating predictions on train/test data\ntrain_predict = resample_model.predict(train_features, batch_size=BATCH_SIZE)\ntest_predict = resample_model.predict(test_features, batch_size=BATCH_SIZE)\n\n\n# Plotting ROC and CM\nplot_roc(\"Train\", train_labels, train_predict)\nplot_roc(\"Test\", test_labels, test_predict)\nplt.show()\nplot_cm(train_labels, train_predict, p=0.6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generating predictions\noutput_prediction = resample_model.predict(predict_features, batch_size=BATCH_SIZE)\n\n# Setting threshold to categorize records\nTHRESHOLD = 0.8\n\npredicted = [1 if x[0] > THRESHOLD else 0 for x in output_prediction.tolist()]\noutput = pd.DataFrame({\n    'id': cleaned_test_df['Id'],\n    'Predicted': predicted\n})\n\noutput.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}