{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "1ad0bbde-8ea9-d1cc-2b58-052284256db8"
      },
      "source": [
        "numeric data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "09a5565c-8938-4381-677a-165df5db654e"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "\n",
        "\n",
        "feature_names = ['L3_S38_F3960', 'L3_S33_F3865', 'L3_S38_F3956', 'L3_S33_F3857',\n",
        "       'L3_S29_F3321', 'L1_S24_F1846', 'L3_S32_F3850', 'L3_S29_F3354',\n",
        "       'L3_S29_F3324', 'L3_S35_F3889', 'L0_S1_F28', 'L1_S24_F1844',\n",
        "       'L3_S29_F3376', 'L0_S0_F22', 'L3_S33_F3859', 'L3_S38_F3952', \n",
        "       'L3_S30_F3754', 'L2_S26_F3113', 'L3_S30_F3759', 'L0_S5_F114']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1509a538-4dae-eb9e-ae37-49d537478c07"
      },
      "outputs": [],
      "source": [
        "numeric_cols = pd.read_csv(\"../input/train_numeric.csv\", nrows = 1).columns.values\n",
        "imp_idxs = [np.argwhere(feature_name == numeric_cols)[0][0] for feature_name in feature_names]\n",
        "train = pd.read_csv(\"../input/train_numeric.csv\", \n",
        "                index_col = 0, header = 0, usecols = [0, len(numeric_cols) - 1] + imp_idxs)\n",
        "train = train[feature_names + ['Response']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "09daf1a3-487d-c9e2-4a52-ed11b5c5a365"
      },
      "outputs": [],
      "source": [
        " X_neg, X_pos = train[train['Response'] == 0].iloc[:, :-1], train[train['Response']==1].iloc[:, :-1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9fc6a8e8-87be-d74b-b414-5924a24884b2"
      },
      "outputs": [],
      "source": [
        "BATCH_SIZE = 5\n",
        "train_batch =[pd.melt(train[train.columns[batch: batch + BATCH_SIZE].append(np.array(['Response']))], \n",
        "                      id_vars = 'Response', value_vars = feature_names[batch: batch + BATCH_SIZE])\n",
        "              for batch in list(range(0, train.shape[1] - 1, BATCH_SIZE))]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5a114b4c-bf3f-84b4-3500-09d6a61aae51"
      },
      "outputs": [],
      "source": [
        "FIGSIZE = (12,16)\n",
        "_, axs = plt.subplots(len(train_batch), figsize = FIGSIZE)\n",
        "plt.suptitle('Univariate distributions')\n",
        "for data, ax in zip(train_batch, axs):\n",
        "    sns.violinplot(x = 'variable',  y = 'value', hue = 'Response', data = data, ax = ax, split =True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c9907c87-23ca-67bb-97e0-ca58d9a55b49"
      },
      "outputs": [],
      "source": [
        "non_missing = pd.DataFrame(pd.concat([(X_neg.count()/X_neg.shape[0]).to_frame('negative samples'),\n",
        "                                      (X_pos.count()/X_pos.shape[0]).to_frame('positive samples'),  \n",
        "                                      ], \n",
        "                       axis = 1))\n",
        "non_missing_sort = non_missing.sort_values(['negative samples'])\n",
        "non_missing_sort.plot.barh(title = 'Proportion of non-missing values', figsize = FIGSIZE)\n",
        "plt.gca().invert_yaxis()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "814714ae-08d6-5ff9-b2b1-c93fa8428fda"
      },
      "outputs": [],
      "source": [
        "MIN_PERIODS = 100\n",
        "\n",
        "triang_mask = np.zeros((X_pos.shape[1], X_pos.shape[1]))\n",
        "triang_mask[np.triu_indices_from(triang_mask)] = True\n",
        "\n",
        "ax1.set_title('Negative Class')\n",
        "sns.heatmap(X_neg.corr(min_periods = MIN_PERIODS), mask = triang_mask, square=True,  ax = ax1)\n",
        "\n",
        "ax2.set_title('Positive Class')\n",
        "sns.heatmap(X_pos.corr(min_periods = MIN_PERIODS), mask = triang_mask, square=True,  ax = ax2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dfb5d22b-c002-2797-2ba3-06771aadb023"
      },
      "outputs": [],
      "source": [
        "sns.heatmap(X_pos.corr(min_periods = MIN_PERIODS) -X_neg.corr(min_periods = MIN_PERIODS), \n",
        "             mask = triang_mask, square=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d6257f65-b157-b630-f758-81ee0f37b5e9"
      },
      "outputs": [],
      "source": [
        "nan_pos, nan_neg = np.isnan(X_pos), np.isnan(X_neg)\n",
        "\n",
        "triang_mask = np.zeros((X_pos.shape[1], X_pos.shape[1]))\n",
        "triang_mask[np.triu_indices_from(triang_mask)] = True\n",
        "\n",
        "FIGSIZE = (13,4)\n",
        "_, (ax1, ax2) = plt.subplots(1,2, figsize = FIGSIZE)\n",
        "MIN_PERIODS = 100\n",
        "\n",
        "ax1.set_title('Negative Class')\n",
        "sns.heatmap(nan_neg.corr(),   square=True, mask = triang_mask, ax = ax1)\n",
        "\n",
        "ax2.set_title('Positive Class')\n",
        "sns.heatmap(nan_pos.corr(), square=True, mask = triang_mask,  ax = ax2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "afd67a2c-298e-9907-d281-29812e3c9a71"
      },
      "outputs": [],
      "source": [
        "sns.heatmap(nan_neg.corr() - nan_pos.corr(), mask = triang_mask, square=True)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}