{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "66fabb4c-2a2c-6ad0-3447-1ac095217ca2" }, "source": [ "Checking the 'train_dates.csv'\n", "\n", " - lots of columns (1157)\n", " - 80%+ missing values\n", " - Same stations often have same date values" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "59539376-020b-b14f-e9f8-6cb4225f35cb" }, "outputs": [], "source": [ "%matplotlib inline\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "train_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\n", "print(train_date_part.shape)\n", "print(1.0 * train_date_part.count().sum() / train_date_part.size)\n", "print(train_date_part[:2])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0f72ecf0-c05a-685e-eba8-fb2316e3aea8" }, "outputs": [], "source": [ "# Let's check the min and max times for each station\n", "def get_station_times(dates, withId=False):\n", " times = []\n", " cols = list(dates.columns)\n", " if 'Id' in cols:\n", " cols.remove('Id')\n", " for feature_name in cols:\n", " if withId:\n", " df = dates[['Id', feature_name]].copy()\n", " df.columns = ['Id', 'time']\n", " else:\n", " df = dates[[feature_name]].copy()\n", " df.columns = ['time']\n", " df['station'] = feature_name.split('_')[1][1:]\n", " df = df.dropna()\n", " times.append(df)\n", " return pd.concat(times)\n", "\n", "station_times = get_station_times(train_date_part, withId=True).sort_values(by=['Id', 'station'])\n", "print(station_times[:5])\n", "print(station_times.shape)\n", "min_station_times = station_times.groupby(['Id', 'station']).min()['time']\n", "max_station_times = station_times.groupby(['Id', 'station']).max()['time']\n", "print(np.mean(1. * (min_station_times == max_station_times)))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "61e4c1db-00fc-06c2-e1ba-d7f087682738" }, "source": [ "We just removed the missing values. As the observation times are almost always unique for staion, Id pair we could spare a lot of memory by reading only one time for each station.\n", "\n", "Please note we checked only 1% of the dataset!" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2cbc563a-3a6b-9a07-475a-3dc658c52b54" }, "outputs": [], "source": [ "# Read station times for train and test\n", "date_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\n", "date_cols" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0eb843bc-30d2-61dc-dc30-70225430787e" }, "outputs": [], "source": [ "fig = plt.figure()\n", "plt.plot(train_time_cnt['time'].values, train_time_cnt['cnt'].values, 'b.', alpha=0.1, label='train')\n", "plt.plot(test_time_cnt['time'].values, test_time_cnt['cnt'].values, 'r.', alpha=0.1, label='test')\n", "plt.title('Original date values')\n", "plt.ylabel('Number of records')\n", "plt.xlabel('Time')\n", "fig.savefig('original_date_values.png', dpi=300)\n", "plt.show()\n", "\n", "print((train_time_cnt['time'].min(), train_time_cnt['time'].max()))\n", "print((test_time_cnt['time'].min(), test_time_cnt['time'].max()))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8c2f40d9-5950-99a5-0395-f8132d97a02e" }, "source": [ "A few observations:\n", "\n", " 1. Train and test set has the same time period\n", " 2. There is a clear periodic pattern \n", " 3. The dates are transformed to 0 - 1718 with granularity of 0.01\n", " 4. There is a gap in the middle\n", "\n", "Could we figure out what does 0.01 mean? Let's check a few auto correlations!" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c29e43b4-a667-5ca8-3b43-39b3a83a9077" }, "outputs": [], "source": [ "time_ticks = np.arange(train_time_cnt['time'].min(), train_time_cnt['time'].max() + 0.01, 0.01)\n", "time_ticks = pd.DataFrame({'time': time_ticks})\n", "time_ticks = pd.merge(time_ticks, train_time_cnt, how='left', on='time')\n", "time_ticks = time_ticks.fillna(0)\n", "# Autocorrelation\n", "x = time_ticks['cnt'].values\n", "max_lag = 8000\n", "auto_corr_ks = range(1, max_lag)\n", "auto_corr = np.array([1] + [np.corrcoef(x[:-k], x[k:])[0, 1] for k in auto_corr_ks])\n", "fig = plt.figure()\n", "plt.plot(auto_corr, 'k.', label='autocorrelation by 0.01')\n", "plt.title('Train Sensor Time Auto-correlation')\n", "period = 25\n", "auto_corr_ks = list(range(period, max_lag, period))\n", "auto_corr = np.array([1] + [np.corrcoef(x[:-k], x[k:])[0, 1] for k in auto_corr_ks])\n", "plt.plot([0] + auto_corr_ks, auto_corr, 'go', alpha=0.5, label='strange autocorrelation at 0.25')\n", "period = 1675\n", "auto_corr_ks = list(range(period, max_lag, period))\n", "auto_corr = np.array([1] + [np.corrcoef(x[:-k], x[k:])[0, 1] for k in auto_corr_ks])\n", "plt.plot([0] + auto_corr_ks, auto_corr, 'ro', markersize=10, alpha=0.5, label='one week = 16.75?')\n", "plt.xlabel('k * 0.01 - autocorrelation lag')\n", "plt.ylabel('autocorrelation')\n", "plt.legend(loc=0)\n", "fig.savefig('train_time_auto_correlation.png', dpi=300)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "dca1f4be-8077-f181-1e1e-3db2dd1a2d48" }, "source": [ "The largest peaks are at approximately 1680 ticks. Let's call it a week ;) \n", "\n", "In each week we could see 7 local maxima ~ days.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3ee876be-a3d9-e141-eb21-0c361ce0a5a7" }, "outputs": [], "source": [ "week_duration = 1679\n", "train_time_cnt['week_part'] = ((train_time_cnt['time'].values * 100) % week_duration).astype(np.int64)\n", "# Aggregate weekly stats\n", "train_week_part = train_time_cnt.groupby(['week_part'])[['cnt']].sum().reset_index()\n", "fig = plt.figure()\n", "plt.plot(train_week_part.week_part.values, train_week_part.cnt.values, 'b.', alpha=0.5, label='train count')\n", "y_train = train_week_part['cnt'].rolling(window=20, center=True).mean().values\n", "plt.plot(train_week_part.week_part.values, y_train, 'b-', linewidth=4, alpha=0.5, label='train count smooth')\n", "plt.title('Relative Part of week')\n", "plt.ylabel('Number of records')\n", "plt.xlim(0, 1680)\n", "fig.savefig('week_duration.png', dpi=300)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0 }