{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f6a25d5b-1478-1b94-93cf-eb38507c1131" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "# print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "import io\n", "import math\n", "from operator import itemgetter\n", "\n", "# Any results you write to the current directory are saved as output.\n", "\n", "\n", "def parse_feature_id(feat_id):\n", " words = feat_id.split('_')\n", " line_num = int(words[0][1:])\n", " station_num = int(words[1][1:])\n", " feat_num = int(words[2][1:])\n", " return (line_num, station_num, feat_num)\n", "\n", "\n", "def get_vector_of_visited_stations(row, *station_numbers):\n", " vec = {st_num: 0 for st_num in station_numbers}\n", " \n", " fields_dict = row.to_dict()\n", " Id = fields_dict.pop('Id', None)\n", " Response = fields_dict.pop('Response', None)\n", " \n", " for field_name, field_value in fields_dict.items():\n", " if not np.isnan(field_value):\n", " station_num = parse_feature_id(field_name)[1]\n", " vec[station_num] = 1\n", " \n", " vec = sorted(vec.items(), key = itemgetter(0))\n", " \n", " vec = tuple([tup[1] for tup in vec])\n", " \n", " return pd.Series({'id': Id, 'label': Response, 'sequence': vec})\n", " \n", " \n", "def get_list_of_nonempty_fields(row):\n", " fields_dict = row.to_dict()\n", "\n", " Id = fields_dict.pop('Id', None)\n", " Response = fields_dict.pop('Response', None)\n", "\n", " nonempty_feature_names = []\n", " for field_name, field_value in fields_dict.items():\n", " if not np.isnan(field_value): nonempty_feature_names.append(parse_feature_id(field_name))\n", "\n", " # convert list of feature names into a tuple to make it hashable\n", " sequence_tuple = tuple(sorted(nonempty_feature_names))\n", " return pd.Series({'id': Id, 'label': Response, 'sequence': sequence_tuple})\n", "\n", "\n", "def get_rid_of_feats(line_stat_feat):\n", " line_stat = [(l, s) for (l, s, f) in line_stat_feat]\n", " uniq_line_stat = set(line_stat)\n", " return tuple(sorted(uniq_line_stat))\n", "\n", "\n", "def get_list_of_stations(col_names):\n", " st_numbers = [parse_feature_id(col_name)[1] for col_name in col_names]\n", " uniq_st_numbers = list(set(st_numbers))\n", " return sorted(uniq_st_numbers)\n", "\n", "num_filename = '../input/train_numeric.csv'\n", "cat_filename = '../input/train_categorical.csv'\n", "\n", "example_data = ('one,two,three,four,Id\\n'\n", " '1,,3,4,1\\n'\n", " '2,20,,40,2\\n'\n", " '100,200,300,400,3\\n'\n", " '1000,,3000,,4\\n'\n", " '7,,90,90,5\\n'\n", " '8,,8,8,6\\n'\n", " '5,,5,,7\\n'\n", " '53,53,,53,8')\n", "\n", "\n", "buf = io.StringIO(example_data)\n", "\n", "# df = pd.read_csv(buf, header=0, dtype=np.float64)\n", "\n", "#print(df)\n", "#print(df.applymap(lambda x: np.isnan(x) if isinstance(x, float) else False))\n", "\n", "num_df = pd.read_csv(num_filename, header=0, nrows=20000, dtype=np.float64)\n", "cat_df = pd.read_csv(cat_filename, header=0, nrows=20000, dtype=str)\n", "\n", "# change Id column type to float:\n", "cat_df[\"Id\"] = cat_df[\"Id\"].astype(np.float64)\n", "\n", " \n", "cat_df = cat_df.applymap(lambda x: int(x[1:]) if isinstance(x, str) and x[0] == 'T' else x)\n", "\n", "\n", "df = num_df.merge(cat_df, on='Id', how='inner')\n", "\n", "print(num_df.shape)\n", "print(cat_df.shape)\n", "print(df.shape)\n", "\n", "\n", "col_names = list(df.columns.values)\n", "if 'Id' in col_names: col_names.remove('Id')\n", "if 'Response' in col_names: col_names.remove('Response')\n", " \n", "list_of_stations = get_list_of_stations(col_names)\n", "\n", "# to strings of 1's and 0's\n", "nonempty_df = df.apply(func = get_vector_of_visited_stations, axis = 1, args = (list_of_stations))\n", "\n", "# to lists of visited stations:\n", "# nonempty_df = df.apply(func = get_list_of_nonempty_fields, axis = 1)\n", "# nonempty_df['feature_ids'] = nonempty_df['feature_ids'].apply(get_rid_of_feats)\n", "\n", "seq_df = nonempty_df.groupby('sequence')\n", "\n", "# for name, group in seq_df:\n", "# print(\"NAME:\")\n", "# print(name)\n", "# print(\"GROUP:\")\n", "# print(\"////////////////////\")\n", "# print(group)\n", "# print(group.__class__.__name__)\n", "# print(\"////////////////////\")\n", "# print(\"^^^^\")\n", "# print()\n", "\n", "groups_df = seq_df['id'].apply(func=list)\n", "\n", "print()\n", "print(groups_df.head(5))\n", "print()\n", "print(groups_df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "af0f8bbf-84dd-e751-e1fd-ff334960bc65" }, "outputs": [], "source": [ "# cluster sequence vectors\n", "\n", "from sklearn.cluster import KMeans\n", "\n", "num_clusters = 5\n", "kmeans = KMeans(n_clusters = num_clusters)\n", "\n", "sequence_vectors = np.array(sorted(groups_df.axes[0].tolist()))\n", "kmeans.fit(sequence_vectors)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5971ff38-4de1-9a09-f1d1-a7a617eb45e4" }, "outputs": [], "source": [ "# create vector column of cluster labels\n", "labels_col = np.array(kmeans.labels_)[:, np.newaxis]\n", "\n", "# add 1 to every label so there is no zeros\n", "labels_col += 1\n", "\n", "print(sequence_vectors.shape)\n", "print(labels_col.shape)\n", "\n", "labeled_vectors = np.multiply(sequence_vectors, labels_col)\n", "\n", "print(labeled_vectors.shape)\n", "\n", "print(labeled_vectors[:5])\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c861a2c3-0957-ce91-ece5-5c5f7097a1a2" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "#fig = plt.figure(figsize=(5, 5))\n", "#ax = fig.add_subplot(111)\n", "\n", "fig = plt.figure(figsize = (15,8))\n", "ax = fig.add_subplot(111)\n", "\n", "ax.matshow(labeled_vectors, aspect='auto')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6d9b4316-48b5-dd0d-1a55-f950704b2410" }, "outputs": [], "source": [ "# let's count how many samples there are in each cluster\n", "\n", "def get_cluster(row, *station_numbers):\n", " row = get_vector_of_visited_stations(row, *station_numbers)\n", " vec = np.array(row['sequence'])\n", " cluster = kmeans.predict([vec])[0]\n", "# return row.append(pd.Series({'cluster': cluster}))\n", " return pd.Series({'cluster': cluster, 'id': row['id']})\n", " \n", "clusters_df = df.apply(func = get_cluster, axis = 1, args = (list_of_stations))\n", "print(\"Done\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f9779594-fb16-1100-3bbe-e36e696d8d9b" }, "outputs": [], "source": [ "counts_df = clusters_df.groupby('cluster').count()\n", "print(counts_df)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0 }