{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7fc5a551-6e27-99fa-ce36-c6550065af60"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "691a93cb-971b-3f84-f775-7385296a4b78"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from scipy.stats.mstats import normaltest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "07602016-0f79-8ba5-7df8-95d92371a1f1"
      },
      "outputs": [],
      "source": [
        "train_numeric_part = pd.read_csv('../input/train_numeric.csv', nrows=10000)\n",
        "header = train_numeric_part.columns.tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "08d7da2d-51e5-46f8-92f8-18aeccc87147"
      },
      "outputs": [],
      "source": [
        "def file_len(fname):\n",
        "    with open(fname) as f:\n",
        "        for i, l in enumerate(f):\n",
        "            pass\n",
        "    return i + 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2fdbe2f0-5ef6-6abd-0e77-d4f4f07e1bca"
      },
      "outputs": [],
      "source": [
        "nline = file_len('../input/train_numeric.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cc856336-b3c1-010b-3031-80289f7b0f81"
      },
      "outputs": [],
      "source": [
        "col = []\n",
        "train_numeric_part_gauss = pd.DataFrame(index=train_numeric_part.index)\n",
        "train_numeric_part_gauss_X = pd.DataFrame(index=train_numeric_part.index)\n",
        "for c in train_numeric_part.columns:\n",
        "    if train_numeric_part[c].dropna().size >= 0.946*train_numeric_part.shape[0]:\n",
        "        if normaltest(train_numeric_part[c].dropna()).pvalue < 0.05:\n",
        "            train_numeric_part_gauss[c] = train_numeric_part[c]\n",
        "            col.append(c)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6161d757-9ef8-25af-c74f-1c4fd6eb815d"
      },
      "outputs": [],
      "source": [
        "train_numeric_part_gauss[\"Response\"] = train_numeric_part['Response']\n",
        "train_numeric_part_gauss = train_numeric_part_gauss.dropna()\n",
        "train_numeric_part_gauss.set_index(\"Id\",inplace=True)\n",
        "train_numeric_part_gauss_X = train_numeric_part_gauss.loc[:,train_numeric_part_gauss.columns != \"Response\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d9327e44-2e10-0791-b530-9f6eb4fc41a9"
      },
      "outputs": [],
      "source": [
        "gnb = GaussianNB()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "350dadd3-cde8-da3c-7586-81ad01968747"
      },
      "outputs": [],
      "source": [
        "gnb = gnb.fit(train_numeric_part_gauss_X,train_numeric_part_gauss.Response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a18a79ab-98a0-eb19-f1a7-b3ec6471a33c"
      },
      "outputs": [],
      "source": [
        "reader = pd.read_csv('../input/train_numeric.csv', chunksize=10000)\n",
        "\n",
        "for chunk in reader:\n",
        "    train_numeric_part_gauss = chunk[col]\n",
        "    train_numeric_part_gauss[\"Response\"] = chunk['Response']\n",
        "    train_numeric_part_gauss = train_numeric_part_gauss.dropna()\n",
        "    train_numeric_part_gauss.set_index(\"Id\",inplace=True)\n",
        "    train_numeric_part_gauss_X = train_numeric_part_gauss.loc[:,train_numeric_part_gauss.columns != \"Response\"]\n",
        "    gnb = gnb.partial_fit(train_numeric_part_gauss_X,train_numeric_part_gauss.Response, classes=np.unique(train_numeric_part_gauss.Response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e4039079-1c9c-126b-0a1d-9530afc17ecd"
      },
      "outputs": [],
      "source": [
        "file_len('../input/test_numeric.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b4cc109f-c925-19cc-9918-4e01245b9dbc"
      },
      "outputs": [],
      "source": [
        "test_numeric_part = pd.read_csv('../input/test_numeric.csv',nrows=1000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e6c8a748-b1b3-cb97-4d06-ceb3de0a089c"
      },
      "outputs": [],
      "source": [
        "res = pd.DataFrame(columns=[\"Id\",\"Response\"])\n",
        "res.set_index(\"Id\",inplace=True)\n",
        "with open('res.csv', 'a') as f:\n",
        "             res.to_csv(f, header=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ac96ab74-80ea-c596-d18a-6b0def2be5a0"
      },
      "outputs": [],
      "source": [
        "reader = pd.read_csv('../input/test_numeric.csv', chunksize=10000)\n",
        "#col.remove(\"Response\")\n",
        "for chunk in reader:\n",
        "    indata = chunk[col]\n",
        "    indata.set_index(\"Id\",inplace=True)\n",
        "    indata = indata.dropna()\n",
        "    indata['res'] = gnb.predict(indata)\n",
        "    with open('res.csv', 'a') as f:\n",
        "             indata[['res']].to_csv(f, header=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4d58070a-c3eb-fecd-5cf4-bd482f5ea362"
      },
      "outputs": [],
      "source": [
        "#print(check_output([\"head\",\"res.csv\"]).decode(\"utf8\"))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "16560e41-83ad-840c-e825-10d54b87de65"
      },
      "outputs": [],
      "source": [
        "res = pd.DataFrame(columns=[\"Id\",\"Response\"])\n",
        "res.set_index(\"Id\",inplace=True)\n",
        "with open('res.csv', 'a') as f:\n",
        "             res.to_csv(f, header=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f5ae94fb-9d81-5547-8ed7-e3ba9709be22"
      },
      "outputs": [],
      "source": [
        "file_len('res.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a7b6dad2-d4fe-6835-7aac-287c875f5187"
      },
      "outputs": [],
      "source": [
        "train_categorical_part = pd.read_csv('../input/test_categorical.csv', nrows=10000, low_memory=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1b3fe27e-5e51-305d-81bd-7e8315fada42"
      },
      "outputs": [],
      "source": [
        "ngcol = []\n",
        "train_numeric_part_ngauss = pd.DataFrame(index=train_numeric_part.index)\n",
        "train_numeric_part_ngauss_X = pd.DataFrame(index=train_numeric_part.index)\n",
        "for c in train_numeric_part.columns:\n",
        "    if train_numeric_part[c].dropna().size >= 0.94*train_numeric_part.shape[0]:\n",
        "        if normaltest(train_numeric_part[c].dropna()).pvalue >= 0.05:\n",
        "            train_numeric_part_ngauss[c] = train_numeric_part[c]\n",
        "            ngcol.append(c)\n",
        "            \n",
        "ngcol.append(\"Response\")\n",
        "ngcol.append(\"Id\")\n",
        "print(ngcol)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8d83a676-94f1-0516-29b9-66ebbcbf09ed"
      },
      "outputs": [],
      "source": [
        "from sklearn import linear_model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f3d5f757-bab8-fad1-09a7-3dee03394ddc"
      },
      "outputs": [],
      "source": [
        "reader = pd.read_csv('../input/train_numeric.csv', chunksize=10000)\n",
        "\n",
        "failcount = 0\n",
        "\n",
        "for chunk in reader:\n",
        "    failcount += chunk.loc[chunk.Response == 1, \"Response\"].size"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3262b55a-0412-627b-3c00-d05bf61b3d3b"
      },
      "outputs": [],
      "source": [
        "s = file_len('../input/test_categorical.csv')\n",
        "cweights = {0 : (s - failcount)/s,\n",
        "           1 : failcount/s}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aba19e67-921a-99f8-1a6b-61d446bcd428"
      },
      "outputs": [],
      "source": [
        "reader = pd.read_csv('../input/train_numeric.csv', chunksize=10000)\n",
        "\n",
        "perc = linear_model.Perceptron(class_weight='balanced')\n",
        "\n",
        "for chunk in reader:\n",
        "    train_numeric_part_ngauss = chunk[ngcol]\n",
        "    train_numeric_part_ngauss[\"Response\"] = chunk['Response']\n",
        "    train_numeric_part_ngauss = train_numeric_part_ngauss.dropna()\n",
        "    train_numeric_part_ngauss.set_index(\"Id\",inplace=True)\n",
        "    train_numeric_part_ngauss_X = train_numeric_part_ngauss.loc[:,train_numeric_part_ngauss.columns != \"Response\"]\n",
        "    perc = perc.partial_fit(train_numeric_part_ngauss_X,train_numeric_part_ngauss.Response, classes=np.unique(train_numeric_part_ngauss.Response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d009459a-ca62-d8c7-69ae-04c9aadd35c8"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}