{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9f51722d-f07a-a375-898c-38b30de1a25b"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn import linear_model\n",
        "\n",
        "train_data = pd.read_csv(\"../input/train.csv\")\n",
        "test_data  = pd.read_csv(\"../input/test.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "63303ae1-7177-1e46-b470-9605bf5c23db"
      },
      "outputs": [],
      "source": [
        "# Partition the data into train and cross-validation\n",
        "\n",
        "print(\"Partitioning training data into cross-validation and training...\")\n",
        "\n",
        "cross_validation_data = train_data.sample(frac = 0.2)\n",
        "train_data_x_cv = train_data.drop(cross_validation_data.index)\n",
        "\n",
        "print(\"Length of original training set \"+str(len(train_data)))\n",
        "print(\"Length of cross-validation set \"+str(len(cross_validation_data)))\n",
        "print(\"Length of final training set \"+str(len(train_data_x_cv)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f15f6c5e-2262-6bb7-8cdf-da0590a6a185"
      },
      "outputs": [],
      "source": [
        "print(train_data.columns)\n",
        "print(test_data.columns)\n",
        "x_variables = list(train_data.columns)\n",
        "x_variables.remove('label')\n",
        "y_variable = 'label'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6c385681-9a26-00e5-62c8-20fe693d4457"
      },
      "outputs": [],
      "source": [
        "# Check if some features are always zero\n",
        "import matplotlib.pyplot as plt\n",
        "temp_df = pd.DataFrame(train_data[x_variables].sum())\n",
        "temp_df.columns = ['sum_of_intensities']\n",
        "#temp_df.reset_index(inplace = True)\n",
        "temp_df.sort_values(by = ['sum_of_intensities'],ascending = [1],inplace = True)\n",
        "plt.figure(figsize=[10,50])\n",
        "temp_df.plot(kind = 'bar')\n",
        "temp_df.reset_index(inplace = True)\n",
        "temp_df.columns = ['features','sum_of_intensities']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c6772317-cf67-a6df-676d-b6c337c2b06e"
      },
      "outputs": [],
      "source": [
        "# Get a list of features for which are always zero and remove them\n",
        "features_to_remove = list(temp_df.loc[temp_df['sum_of_intensities']==0,'features'])\n",
        "print(\"Following are the features that neeed to be removed \\n\")\n",
        "print(features_to_remove)\n",
        "x_variables_copy = x_variables[:]\n",
        "print(\"\\n These features will now be removed \\n\")\n",
        "for f in features_to_remove:\n",
        "    x_variables_copy.remove(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ad396839-b288-707f-071a-5a32bf9b9720"
      },
      "outputs": [],
      "source": [
        "# Neural Netwroks\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.metrics import accuracy_score\n",
        "Y = train_data_x_cv[y_variable].as_matrix()\n",
        "X = train_data_x_cv[x_variables_copy].as_matrix()\n",
        "\n",
        "print(\"Training neural nets... \")\n",
        "my_classifier = MLPClassifier(solver = 'adam',alpha = 0.1,hidden_layer_sizes = (50))\n",
        "my_classifier.fit(X,Y)\n",
        "predicted = my_classifier.predict(cross_validation_data[x_variables_copy].as_matrix())\n",
        "\n",
        "print(\"Getting Accuracy for CV...\")\n",
        "accu = accuracy_score(cross_validation_data[y_variable].as_matrix(),predicted)\n",
        "print(\"Accuracy = \"+str(accu))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e8fda862-7d90-76f6-663d-2ef11b046531"
      },
      "outputs": [],
      "source": [
        "predicted_test_labels = my_classifier.predict(test_data[x_variables_copy])\n",
        "data_dict = {'ImageId':range(1,len(predicted_test_labels)+1),'Label':predicted_test_labels}\n",
        "test_labels = pd.DataFrame(data_dict)\n",
        "test_labels.to_csv(r'Output_neural_nets.csv')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "12a20e21-d864-a631-4674-d5e4e4e76b37"
      },
      "outputs": [],
      "source": [
        "# classify using a logistic classifier with C = 50\n",
        "Y = train_data_x_cv[y_variable]\n",
        "X = train_data_x_cv[x_variables_copy]\n",
        "\n",
        "print(\"Training a logistic regression model ... \")\n",
        "logreg = linear_model.LogisticRegression(penalty = 'l2',C = 50,fit_intercept = True,solver = 'sag',max_iter = 300)\n",
        "logreg.fit(X,Y)\n",
        "predicted = logreg.predict(cross_validation_data[x_variables_copy])\n",
        "\n",
        "print(np.corrcoef(cross_validation_data[y_variable],predicted))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "16bb94be-02e2-6cdc-d87b-f100e9a9b0e5"
      },
      "outputs": [],
      "source": [
        "predicted_test_labels = my_classifier.predict(test_data[x_variables_copy])\n",
        "data_dict = {'ImageId':range(1,len(predicted_test_labels)+1),'Label':predicted_test_labels}\n",
        "test_labels = pd.DataFrame(data_dict)\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}