{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "cd9c3ebf-aacc-a45f-192e-e3e33228610a"
      },
      "source": [
        "This script will one-hot encode categories and save them as scipy.sparse.csc_matrix. It will also generate the header so that you know which columns represent which original columns.\n",
        "The sparse representation will take 275MB on disk and about 800MB in memory."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f318a3d3-6112-3943-da03-0c97525cc80a"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from itertools import chain\n",
        "import scipy.sparse as spar\n",
        "\n",
        "\n",
        "def sparse_dummies(categories):\n",
        "    num_rows = categories.shape[0]\n",
        "    num_cats = len(categories.values.categories)\n",
        "    if num_cats == 0: return None\n",
        "    categories = categories.reset_index(drop=True)[categories.values.codes > -1]\n",
        "    data = np.ones(categories.shape[0])\n",
        "    return spar.csc_matrix(\n",
        "        (data, (categories.index.values, categories.values.codes)),\n",
        "        shape=(num_rows, num_cats))\n",
        "\n",
        "\n",
        "def one_hot(df):\n",
        "    df = df.apply(lambda col: col.astype('category'), axis=0)\n",
        "    cat_counts = df.apply(lambda col: len(col.values.categories), axis=0)\n",
        "    header = list(chain.from_iterable(\n",
        "    [[colname] * ncats for colname, ncats in zip(cat_counts.index, cat_counts.values)]))\n",
        "\n",
        "    dummy_matrices = (sparse_dummies(df[col]) for col in df.columns)\n",
        "    # drop None's\n",
        "    dummy_matrices = list(elem for elem in dummy_matrices if elem is not None)\n",
        "    \n",
        "    if len(dummy_matrices) == 0: return None, None\n",
        "    mtx = spar.hstack(dummy_matrices)\n",
        "    \n",
        "    return header, mtx\n",
        "\n",
        "\n",
        "def save_ohe(filename, header, matrix):\n",
        "    np.savez(filename,\n",
        "        header=header,\n",
        "        indices=matrix.indices,\n",
        "        indptr=matrix.indptr,\n",
        "        shape=matrix.shape)\n",
        "\n",
        "\n",
        "def load_ohe(filename):\n",
        "    loader = np.load(filename)\n",
        "    header = loader['header']\n",
        "    indices = loader['indices']\n",
        "    indptr = loader['indptr']\n",
        "    data = np.ones(len(indices))\n",
        "    mtx = spar.csc_matrix((data, indices, indptr), shape = loader['shape'])\n",
        "    return header, mtx\n",
        "\n",
        "\n",
        "def chunker(seq, size):\n",
        "    return (seq[pos:pos + size] for pos in range(0, len(seq), size))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c0e11d70-bbe8-605a-6750-b8d1daa0445c"
      },
      "outputs": [],
      "source": [
        "cat_path = '../input/train_categorical.csv'\n",
        "# cat_path = '../input/test_categorical.csv'\n",
        "\n",
        "NROWS = 50000 # test on 1st 50 000 rows (the entire dataset will probably take too long for kaggle)\n",
        "\n",
        "chunksize = 30 # number of columns to read at once\n",
        "\n",
        "columns = pd.read_csv(\n",
        "    cat_path,\n",
        "    index_col=0,\n",
        "    nrows = 1,\n",
        "    dtype=str).columns.values\n",
        "\n",
        "ncols = len(columns)\n",
        "print(ncols)\n",
        "\n",
        "headers = []\n",
        "matrices = []\n",
        "progress = 0\n",
        "# read file in vertical chunks\n",
        "for col_subset in chunker(columns, chunksize):\n",
        "    progress += chunksize\n",
        "    \n",
        "    usecols = np.append(['Id'], col_subset)\n",
        "    df = pd.read_csv(\n",
        "        cat_path,\n",
        "        index_col=0,\n",
        "        dtype=str,\n",
        "        nrows=NROWS,\n",
        "        usecols=usecols\n",
        "    )\n",
        "    \n",
        "    header, mtx = one_hot(df)\n",
        "    if mtx is not None:\n",
        "        matrices.append(mtx)\n",
        "        headers.append(header)\n",
        "        print(\"progress = {:.1f}% : {}\".format(100.*progress / (1.*ncols), mtx.shape))\n",
        "\n",
        "\n",
        "full_header = np.hstack(headers)\n",
        "full_mtx = spar.hstack(matrices).tocsc()\n",
        "\n",
        "print(full_header.shape)\n",
        "print(full_mtx.shape)\n",
        "\n",
        "with open('train_cat_sparse', 'wb') as resfile:\n",
        "    save_ohe(resfile, full_header, full_mtx)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "5b5ff0c0-a452-36d6-c314-3948145600ca"
      },
      "source": [
        "You can load the data using the load_ohe method and use it with xgboost:\n",
        "\n",
        "\n",
        "----------\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "461729ba-7b0f-a836-a9ba-28d792fcb36c"
      },
      "outputs": [],
      "source": [
        "header, mtx = load_ohe('train_cat_sparse')\n",
        "\n",
        "print(header.shape)\n",
        "print(mtx.shape)\n",
        "\n",
        "# use this matrix with xgboost\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
        "from sklearn.metrics import matthews_corrcoef, roc_auc_score\n",
        "\n",
        "num_path = '../input/train_numeric.csv'\n",
        "labels = pd.read_csv(num_path, usecols=[969], nrows = NROWS, dtype=np.float32)\n",
        "\n",
        "X = mtx\n",
        "y = labels.values.ravel()\n",
        "y_pred = np.empty(y.shape)\n",
        "\n",
        "clf = XGBClassifier()\n",
        "\n",
        "kfold = StratifiedKFold(n_splits=4, shuffle=True)\n",
        "for (train, test) in kfold.split(X, y):\n",
        "    y_pred[test] = clf.fit(X[train], y[train]).predict_proba(X[test])[:,1]\n",
        "    print(\"ROC AUC = {:.3f}\".format(roc_auc_score(y[test], y_pred[test])))\n",
        "    \n",
        "print()     \n",
        "print(\"FINAL SCORE = {:.3f}\".format(roc_auc_score(y, y_pred)))"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}