{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "02eaf587-125d-a4fe-62c7-3dac63b6c75f"
      },
      "source": [
        "Forked and influenced from [Python XGBoost Starter][1].\n",
        "Added PCA in pipeline with XGBoost Algorithm\n",
        "\n",
        "  [1]: https://www.kaggle.com/joconnor/bosch-production-line-performance/python-xgboost-starter-0-209-public-mcc/notebook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b8ff0786-470f-f3de-e826-2334abc61cda"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.metrics import matthews_corrcoef, roc_auc_score\n",
        "from sklearn.cross_validation import cross_val_score, StratifiedKFold\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.cross_validation import StratifiedKFold\n",
        "\n",
        "from sklearn import preprocessing \n",
        "from sklearn.decomposition import PCA \n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib\n",
        "import seaborn as sns\n",
        "matplotlib.style.use('ggplot') "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eb108407-22f9-9716-1905-d6ebfd6a5b59"
      },
      "outputs": [],
      "source": [
        "# I'm limited by RAM here and taking the first N rows is likely to be\n",
        "# a bad idea for the date data since it is ordered.\n",
        "# Sample the data in a roundabout way:\n",
        "date_chunks = pd.read_csv(\"../input/train_date.csv\", index_col=0, chunksize=100000, dtype=np.float32)\n",
        "num_chunks = pd.read_csv(\"../input/train_numeric.csv\", index_col=0,\n",
        "                         usecols=list(range(969)), chunksize=100000, dtype=np.float32)\n",
        "X = pd.concat([pd.concat([dchunk, nchunk], axis=1).sample(frac=0.05)\n",
        "               for dchunk, nchunk in zip(date_chunks, num_chunks)])\n",
        "y = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, usecols=[0,969], dtype=np.float32).fillna(0).loc[X.index].values.ravel()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4f874b87-60ef-4ea4-4bce-75747762751c"
      },
      "outputs": [],
      "source": [
        "X = X.fillna(0)\n",
        "X = X.values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "fdd06055-aed4-a9a6-986d-a63fccb5aeb8"
      },
      "outputs": [],
      "source": [
        "X3 = X #set varible for code below"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aa384ed8-5b86-2925-18f7-67d9445aa122"
      },
      "outputs": [],
      "source": [
        "# Only need to run once\n",
        "# n components will explain up to 70% of the data\n",
        "#PCA_A = PCA(n_components=X3.shape[1]-1)\n",
        "#PCA_Y = PCA_A.fit_transform(X3)\n",
        "#\n",
        "#print(PCA_A.n_components)\n",
        "#print(PCA_A.explained_variance_[:15])\n",
        "#x_vals = np.arange(1, PCA_A.n_components)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3f71b9c6-b4de-8eb9-c3c9-ab2c28fbcf05"
      },
      "outputs": [],
      "source": [
        "#tmp =0 \n",
        "#for i, p in enumerate(PCA_A.explained_variance_ratio_):\n",
        "#    tmp = sum(PCA_A.explained_variance_ratio_[:i])\n",
        "#    if tmp >=.70 :\n",
        "#        print('up until component #:', i)\n",
        "#       #print np.cumsum(PCA_A.explained_variance_ratio_[:i])\n",
        "#        break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2d9a1e76-c290-9fdd-8028-7520fdf5d4d1"
      },
      "outputs": [],
      "source": [
        "#plt.figure(figsize=(7,5))\n",
        "\n",
        "#plt.plot(range(X3.shape[1]-1), PCA_A.explained_variance_ratio_, '-o', label='Individual component')\n",
        "#plt.plot(range(X3.shape[1]-1), np.cumsum(PCA_A.explained_variance_ratio_), '-s', label='Cumulative')\n",
        "\n",
        "#plt.ylabel('Proportion of Variance Explained')\n",
        "#plt.xlabel('Principal Component')\n",
        "#plt.xlim(0.75, 20)\n",
        "#plt.ylim(0,1.05)\n",
        "#plt.xticks(range(20))\n",
        "#plt.legend(loc=2);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "13082ad8-cc7a-0388-0cf1-b5f41f0c568a"
      },
      "outputs": [],
      "source": [
        "#Create a pipeline for PCA & XGBClassifier\n",
        "pipe_boosch = Pipeline(steps=[('pca', PCA()), \n",
        "                            ('xgb', XGBClassifier()) ])\n",
        "\n",
        "cvs = StratifiedKFold(y, n_folds = 3, shuffle=True)\n",
        "pipe_boosch.set_params(pca__n_components=100, xgb__base_score=0.005)\n",
        "print(cross_val_score(pipe_boosch, X3, y, cv=cvs, n_jobs=-1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7fa0923e-ddfa-d046-f7a8-842667883110"
      },
      "outputs": [],
      "source": [
        "clf = XGBClassifier(base_score=0.005)\n",
        "clf.fit(X3, y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c6159cc6-47b5-e632-b0b6-2f8e0578876d"
      },
      "outputs": [],
      "source": [
        "# threshold for a manageable number of features\n",
        "plt.hist(clf.feature_importances_[clf.feature_importances_>0])\n",
        "important_indices = np.where(clf.feature_importances_>0.001)[0]\n",
        "print(important_indices)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d2a8fa06-d180-d316-2f92-aef473339726"
      },
      "outputs": [],
      "source": [
        "# load entire dataset for these features. \n",
        "# note where the feature indices are split so we can load the correct ones straight from read_csv\n",
        "n_date_features = 1156\n",
        "X_new = np.concatenate([\n",
        "    pd.read_csv(\"../input/train_date.csv\", index_col=0, dtype=np.float32,\n",
        "                usecols=np.concatenate([[0], important_indices[important_indices < n_date_features] + 1])).fillna(0).values,\n",
        "    pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32,\n",
        "                usecols=np.concatenate([[0], important_indices[important_indices >= n_date_features] + 1 - 1156])).fillna(0).values\n",
        "], axis=1)\n",
        "y_new = pd.read_csv(\"../input/train_numeric.csv\", index_col=0, dtype=np.float32, usecols=[0,969]).values.ravel()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0dd2d440-bb20-82c5-7519-d1eb24a7e510"
      },
      "outputs": [],
      "source": [
        "clf_pipeline = Pipeline(steps=[('pca', PCA()), \n",
        "                            ('xgb', XGBClassifier()) ])\n",
        "cvs = StratifiedKFold(y_new, n_folds = 2)\n",
        "preds = np.ones(y_new.shape[0])\n",
        "clf_pipeline.set_params(pca__n_components=20, xgb__base_score=0.005).fit(X_new, y_new)\n",
        "\n",
        "#for i, (train, test) in enumerate(cvs):\n",
        "#    preds[test] = clf_pipeline.fit(X_new[train], y_new[train]).predict_proba(X[test])[:,1]\n",
        "#    print(\"fold {}, ROC AUC: {:.3f}\".format(i, roc_auc_score(y[test], preds[test])))\n",
        "#print(roc_auc_score(y_new, preds))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eca4bd3d-586a-233d-6dcd-4380939e2b1e"
      },
      "outputs": [],
      "source": [
        "# pick the best threshold out-of-fold\n",
        "thresholds = np.linspace(0.01, 0.99, 50)\n",
        "mcc = np.array([matthews_corrcoef(y_new, preds>thr) for thr in thresholds])\n",
        "plt.plot(thresholds, mcc)\n",
        "best_threshold = thresholds[mcc.argmax()]\n",
        "print(mcc.max())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0e9aceeb-3b06-74af-3238-c500a33d5a71"
      },
      "outputs": [],
      "source": [
        "# load test data\n",
        "X_test = np.concatenate([\n",
        "    pd.read_csv(\"../input/test_date.csv\", index_col=0, dtype=np.float32,\n",
        "                usecols=np.concatenate([[0], important_indices[important_indices<1156]+1])).fillna(0).values,\n",
        "    pd.read_csv(\"../input/test_numeric.csv\", index_col=0, dtype=np.float32,\n",
        "                usecols=np.concatenate([[0], important_indices[important_indices>=1156] +1 - 1156])).fillna(0).values\n",
        "], axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a796c10a-d149-0608-350b-6adb5c733e8b"
      },
      "outputs": [],
      "source": [
        "# generate predictions at the chosen threshold\n",
        "preds = (clf_pipeline.predict_proba(X_test)[:,1] > best_threshold).astype(np.int8)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e0574c4a-ece0-6545-fbc9-cafe21d0844c"
      },
      "outputs": [],
      "source": [
        "# and submit\n",
        "sub = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\n",
        "sub[\"Response\"] = preds\n",
        "sub.to_csv(\"submission.csv.gz\", compression=\"gzip\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cf1bf844-355b-abc1-8ce9-af31ef468fa5"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}