{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "8a8842bf-0a4e-76c2-b7f3-6ff47e020723"
      },
      "source": [
        "Feature reduction with Scikit-learn "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0b13b9ec-3f0c-4480-3254-d8ac0347b621"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "train_array = np.genfromtxt(\"../input/train_numeric.csv\",delimiter=',',filling_values=np.NaN,skip_header=1,max_rows=100000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "38404ee6-e8fe-12e6-4e4f-b96c867a86dc"
      },
      "outputs": [],
      "source": [
        "#Check if defect percentage are propotionate in the chosen subset!\n",
        "test_array= np.genfromtxt(\"../input/train_numeric.csv\",delimiter=',',filling_values=np.NaN,skip_header=100001,max_rows=30000)\n",
        "test_rate = 100*sum(test_array[:,-1] ==1 )/len(test_array[:,-1])\n",
        "train_rate= 100*sum(train_array[:,-1] ==1 )/len(train_array[:,-1])\n",
        "print(test_rate,train_rate)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1c76268d-45f1-ab1f-a1da-bbebfd37e6ee"
      },
      "outputs": [],
      "source": [
        "#Set up test/train features, Impute NaN's to values that scikit learn can work with!\n",
        "train_features = train_array[:,1:-1]\n",
        "test_features = test_array[:,1:-1]\n",
        "from sklearn.preprocessing import Imputer\n",
        "imp = Imputer(missing_values='NaN', strategy='mean', axis=0) #Scikit-learn cant deal with NaN's\n",
        "imp.fit(train_features)\n",
        "train_features= imp.transform(train_features)\n",
        "#Test set\n",
        "imp.fit(test_features)\n",
        "test_features=imp.transform(test_features)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "043eacaf-1632-2b3a-a253-7755de08d9bd"
      },
      "outputs": [],
      "source": [
        "#Feature reduction using tree model\n",
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "clf = ExtraTreesClassifier()\n",
        "clf = clf.fit(train_features, train_array[:,-1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "136b73ae-3c6a-20d8-878c-e894f52d693c"
      },
      "outputs": [],
      "source": [
        "#Analyze reduced datasets\n",
        "len(clf.feature_importances_ ) \n",
        "model = SelectFromModel(clf, prefit=True)\n",
        "train_new = model.transform(train_features)\n",
        "test_new= model.transform(test_features)\n",
        "print(train_new.shape) #Shape of the reduced array 968 --> 266"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "814aae03-2102-5643-4b77-25b5c93551d1"
      },
      "outputs": [],
      "source": [
        "#Extract Feature ranks\n",
        "importances = clf.feature_importances_\n",
        "std = np.std([tree.feature_importances_ for tree in clf.estimators_],\n",
        "             axis=0)\n",
        "\n",
        "indices = np.argsort(importances)[::-1] #Sort features on the basis of rank"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7ed3245b-b870-29f0-d126-6a0da4517ebf"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "# Print & plotthe feature ranking\n",
        "print(\"Feature ranking:\")\n",
        "\n",
        "for f in range(train_features.shape[1]):\n",
        "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
        "\n",
        "# Plot the feature importances of the forest\n",
        "plt.figure()\n",
        "plt.title(\"Feature importances\")\n",
        "plt.bar(range(train_features.shape[1]), importances[indices],\n",
        "       color=\"r\", yerr=std[indices], align=\"center\")\n",
        "plt.xticks(range(train_features.shape[1]), indices)\n",
        "plt.xlim([-1, train_features.shape[1]])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "657f827d-febe-0ce9-1bae-9b1571e7e841"
      },
      "outputs": [],
      "source": [
        "#Fit the new model with reduced features \n",
        "clf_rev = ExtraTreesClassifier()\n",
        "clf_rev = clf_rev.fit(train_new, train_array[:,-1])\n",
        "\n",
        "#make predictions using reduced model\n",
        "prediction = clf_rev.predict(test_new)\n",
        "print(prediction == test_array[:,-1]) \n",
        "success = 100.*sum(prediction == test_array[:,-1])/float(len(test_array[:,-1]))\n",
        "print(\"\\n The  predictions are  %.4f %% accurate\" %(success) )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "078a52ec-3cd8-83a4-a918-13484361c050"
      },
      "outputs": [],
      "source": [
        "#Compute MCC to check the quality of prediction\n",
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_corrcoef(test_array[:,-1],prediction )  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2df8bf3e-0525-2931-d355-b281caaf58d7"
      },
      "outputs": [],
      "source": [
        "clear"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a9dd0deb-b4c1-c4d3-0ce9-85e826b963f3"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}