{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "9e0c5d8f-e48b-e45c-3e7d-29fcce0fceb2" }, "source": [ "#Fail rates vs. flow paths\n", "\n", "Let's find all unique flow paths and calculate failure rates for them" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f5ab7818-e74b-8897-898c-0fc3c2234b8e" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ae778cb5-3041-714b-83c5-fd1697dfed2f" }, "outputs": [], "source": [ "def station_number(col_name):\n", " return int(col_name.split('_')[1][1:])\n", "\n", "\n", "def get_combination(row):\n", "# feats = [(feat_name, feat_exists) for feat_name, feat_exists in row.items()\\\n", "# if feat_name not in ['Id', 'Response']]\n", " combin = [0 for _ in range(52)]\n", " for feat_name, feat_exists in row.items():\n", " if feat_exists:\n", " st_num = station_number(feat_name)\n", " combin[st_num] = 1\n", "\n", " # make into a string\n", " combin_str = ''\n", " for val in combin: combin_str += str(val)\n", " return combin_str\n", "\n", "\n", "def write_csv(streamobj, data):\n", " for row in data:\n", " line = str(row[0])\n", " for word in row[1:]: line += ',' + str(word)\n", " streamobj.write(line + '\\n')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "70f0050a-6caf-6918-6ad9-b31a9d50ca97" }, "source": [ "#Let's read data and collect all unique combinations\n", "#WARNING: This will take a long time" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1d527b2f-29cf-1599-8a76-700858363995" }, "outputs": [], "source": [ "num_path = '../input/train_numeric.csv'\n", "cat_path = '../input/train_categorical.csv'\n", "\n", "NROWS = 550000 # total number of rows to read\n", "CHUNK = 10000\n", "uniq_combinations = set()\n", "\n", "all_ids = []\n", "all_responses = []\n", "all_combinations = []\n", "\n", "lines_read = 0\n", "for num_df, cat_df in zip(\n", " pd.read_csv(num_path, dtype=str, chunksize=CHUNK),\n", " pd.read_csv(cat_path, dtype=str, chunksize=CHUNK)\n", "):\n", " all_ids.extend(num_df['Id'].values)\n", " all_responses.extend(num_df['Response'].values)\n", " \n", " num_df = num_df.notnull().drop(['Id', 'Response'], axis=1)\n", " cat_df = cat_df.notnull().drop(['Id'], axis=1)\n", " for numrow, catrow in zip(num_df.iterrows(), cat_df.iterrows()): \n", " full_row = numrow[1].to_dict()\n", " full_row.update(catrow[1].to_dict())\n", " combination = get_combination(full_row)\n", " uniq_combinations.add(combination)\n", " all_combinations.append(combination)\n", " \n", " lines_read += CHUNK\n", " \n", " if lines_read % 100000 == 0: print(\"progress: {0}\".format(lines_read))\n", " if lines_read >= NROWS: break\n", "\n", "uniq_combinations = list(enumerate(sorted(uniq_combinations)))\n", "print(\"found {0} combinations\".format(len(uniq_combinations)))\n", "\n", " \n", "with open(\"enumerated_combinations.csv\", 'w') as resfile:\n", " resfile.write(\"combid,combination\\n\")\n", " write_csv(resfile, uniq_combinations)\n", " \n", "with open(\"id-comb-res.csv\", 'w') as resfile:\n", " resfile.write(\"Id,combination,Response\\n\")\n", " write_csv(resfile, zip(all_ids, all_combinations, all_responses))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "112e409f-6c6d-6607-1c42-db18307fede5" }, "outputs": [], "source": [ "test_df = pd.read_csv(\"enumerated_combinations.csv\")\n", "print(test_df.head(5))\n", "\n", "test_df = pd.read_csv(\"id-comb-res.csv\")\n", "print(test_df.head(5))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "79cf91b6-7c84-889d-4f1d-d9c5b84c393b" }, "source": [ "Now we'll calculate failure rates for each combid, but will only plot the significant ones (with more than 100 samples). We'll also write fail rates and sample counts as csv." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "48093b43-796d-aa9f-ee18-8b4217f71a49" }, "outputs": [], "source": [ "enum_comb_path = \"enumerated_combinations.csv\"\n", "id_comb_res_path = \"id-comb-res.csv\"\n", "\n", "SIGNIF = 100 # significance threshold\n", "\n", "# create dictionary {combination: combid} e.g. {'1110100110010000000000000000011101110100000000000000': 6779}\n", "combid_dic = pd.read_csv(enum_comb_path, index_col=1).to_dict()['combid']\n", "data_df = pd.read_csv(id_comb_res_path, dtype=str)\n", "\n", "data_df['combid'] = data_df['combination'].apply(lambda comb_str: combid_dic[comb_str])\n", "\n", "responses = data_df['Response'].astype(float).values\n", "combids = data_df['combid'].values\n", "\n", "\n", "# PROCESSING DATA\n", "#############################################################################\n", "from operator import itemgetter\n", "from itertools import groupby\n", "\n", "fail_combids = [combid for combid, resp in zip(combids, responses) if resp > 0]\n", "\n", "all_combids = sorted(combids)\n", "fail_combids = sorted(fail_combids)\n", "\n", "all_counts\t\t= [(combid, len(list(group))) for combid, group in groupby(all_combids)]\n", "failure_counts\t= [(combid, len(list(group))) for combid, group in groupby(fail_combids)]\n", "\n", "failure_counts = dict(failure_counts)\n", "\n", "# throw away rare insignificant combids:\n", "significant_counts = list(filter(lambda combid_count: combid_count[1] >= SIGNIF, all_counts))\n", "\n", "failure_rates_sig = [(combid, failure_counts.get(combid, 0) / (1.*num_samples))\\\n", " for combid, num_samples in significant_counts]\n", "\n", "failure_rates_all = [(combid, failure_counts.get(combid, 0) / (1.*num_samples))\\\n", " for combid, num_samples in all_counts]\n", "#############################################################################\n", "\n", "\n", "\n", "# let's write down failure rates for all significant combids sorted from high to low\n", "failure_rates_sig = sorted(failure_rates_sig, key=itemgetter(1), reverse=True)\n", "with open('failure_rates.csv', 'w') as resfile:\n", " resfile.write(\"combid,failrate\\n\")\n", " write_csv(resfile, failure_rates_sig)\n", " \n", "# let's write down sample counts for all combids\n", "all_counts = sorted(all_counts, key=itemgetter(1), reverse=True)\n", "with open('sample_counts.csv', 'w') as resfile:\n", " resfile.write(\"combid,count\\n\")\n", " write_csv(resfile, all_counts)\n", " \n", "print(\"{0} points total\".format(len(all_counts)))\n", "print(\"{0} significant points\".format(len(significant_counts)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9df94f16-ffdc-fa4f-3364-15d075110594" }, "outputs": [], "source": [ "test_df = pd.read_csv(\"failure_rates.csv\")\n", "print(test_df.head(5))\n", "\n", "test_df = pd.read_csv(\"sample_counts.csv\")\n", "print(test_df.head(5))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "69459338-63cb-5556-f8e9-7b128653678e" }, "source": [ "Let's plot the failure rates for significant combids and numbers of samples for all combids" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "05f2885a-3c5a-e487-144e-45b6da1e8371" }, "outputs": [], "source": [ "# PLOTTING\n", "#############################################################################\n", "from matplotlib import pyplot as plt\n", "import math\n", "\n", "failure_rates_all\t= np.array(failure_rates_all)\n", "failure_rates_sig\t= np.array(failure_rates_sig)\n", "all_counts\t\t\t= np.array(all_counts)\n", "\n", "# plot failrates only for significant combids\n", "fig2, ax2 = plt.subplots(figsize=(16,8))\n", "ax2.bar(failure_rates_sig[:,0], failure_rates_sig[:,1], facecolor='red', edgecolor='red')\n", "ax2.set_ylabel(\"failure rate\")\n", "ax2.set_xlabel(\"combid\")\n", "fig2.savefig(\"failures-vs-combids.png\")\n", "\n", "\n", "# plot numbers of samples for each combid\n", "veclog10 = np.vectorize(lambda x: math.log(x, 10))\n", "\n", "fig3, ax3 = plt.subplots(figsize=(16,8))\n", "ax3.bar(all_counts[:,0], veclog10(all_counts[:,1]), facecolor='black', edgecolor='black')\n", "ax3.set_xlabel('combid')\n", "ax3.set_ylabel('$log_{10}$(number of samples)')\n", "fig3.savefig(\"logcounts-vs-combids.png\")\n", "#############################################################################" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0 }