{"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5357,"databundleVersionId":38056,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Checking the 'train_dates.csv'\n\n - lots of columns (1157)\n - 80%+ missing values\n - Same stations often have same date values","metadata":{"_cell_guid":"66fabb4c-2a2c-6ad0-3447-1ac095217ca2"}},{"cell_type":"code","source":"%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ntrain_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\nprint(train_date_part.shape)\nprint(1.0 * train_date_part.count().sum() / train_date_part.size)\nprint(train_date_part[:2])","metadata":{"_cell_guid":"59539376-020b-b14f-e9f8-6cb4225f35cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's check the min and max times for each station\ndef get_station_times(dates, withId=False):\n    times = []\n    cols = list(dates.columns)\n    if 'Id' in cols:\n        cols.remove('Id')\n    for feature_name in cols:\n        if withId:\n            df = dates[['Id', feature_name]].copy()\n            df.columns = ['Id', 'time']\n        else:\n            df = dates[[feature_name]].copy()\n            df.columns = ['time']\n        df['station'] = feature_name.split('_')[1][1:]\n        df = df.dropna()\n        times.append(df)\n    return pd.concat(times)\n\nstation_times = get_station_times(train_date_part, withId=True).sort_values(by=['Id', 'station'])\nprint(station_times[:5])\nprint(station_times.shape)\nmin_station_times = station_times.groupby(['Id', 'station']).min()['time']\nmax_station_times = station_times.groupby(['Id', 'station']).max()['time']\nprint(np.mean(1. * (min_station_times == max_station_times)))","metadata":{"_cell_guid":"0f72ecf0-c05a-685e-eba8-fb2316e3aea8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We just removed the missing values. As the observation times are almost always unique for staion, Id pair we could spare a lot of memory by reading only one time for each station.\n\nPlease note we checked only 1% of the dataset!","metadata":{"_cell_guid":"61e4c1db-00fc-06c2-e1ba-d7f087682738"}},{"cell_type":"code","source":"# Read station times for train and test\ndate_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\ndate_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\ndate_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\nprint(date_cols) # selected features\n\ntrain_date = pd.read_csv('../input/train_date.csv', usecols=date_cols)\nprint(train_date.shape)\ntrain_station_times = get_station_times(train_date, withId=False)\nprint(train_station_times.shape)\ntrain_time_cnt = train_station_times.groupby('time').count()[['station']].reset_index()\ntrain_time_cnt.columns = ['time', 'cnt']\nprint(train_time_cnt.shape)\n\ntest_date = pd.read_csv('../input/test_date.csv', usecols=date_cols)\nprint(test_date.shape)\ntest_station_times = get_station_times(test_date, withId=False)\nprint(test_station_times.shape)\ntest_time_cnt = test_station_times.groupby('time').count()[['station']].reset_index()\ntest_time_cnt.columns = ['time', 'cnt']\nprint(test_time_cnt.shape)","metadata":{"_cell_guid":"2cbc563a-3a6b-9a07-475a-3dc658c52b54"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure()\nplt.plot(train_time_cnt['time'].values, train_time_cnt['cnt'].values, 'b.', alpha=0.1, label='train')\nplt.plot(test_time_cnt['time'].values, test_time_cnt['cnt'].values, 'r.', alpha=0.1, label='test')\nplt.title('Original date values')\nplt.ylabel('Number of records')\nplt.xlabel('Time')\nfig.savefig('original_date_values.png', dpi=300)\nplt.show()\n\nprint((train_time_cnt['time'].min(), train_time_cnt['time'].max()))\nprint((test_time_cnt['time'].min(), test_time_cnt['time'].max()))","metadata":{"_cell_guid":"0eb843bc-30d2-61dc-dc30-70225430787e"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A few observations:\n\n 1. Train and test set has the same time period\n 2. There is a clear periodic pattern \n 3. The dates are transformed to 0 - 1718 with granularity of 0.01\n 4. There is a gap in the middle\n\nCould we figure out what does 0.01 mean?  Let's check a few auto correlations!","metadata":{"_cell_guid":"8c2f40d9-5950-99a5-0395-f8132d97a02e"}},{"cell_type":"code","source":"time_ticks = np.arange(train_time_cnt['time'].min(), train_time_cnt['time'].max() + 0.01, 0.01)\ntime_ticks = pd.DataFrame({'time': time_ticks})\ntime_ticks = pd.merge(time_ticks, train_time_cnt, how='left', on='time')\ntime_ticks = time_ticks.fillna(0)\n# Autocorrelation\nx = time_ticks['cnt'].values\nmax_lag = 8000\nauto_corr_ks = range(1, max_lag)\nauto_corr = np.array([1] + [np.corrcoef(x[:-k], x[k:])[0, 1] for k in auto_corr_ks])\nfig = plt.figure()\nplt.plot(auto_corr, 'k.', label='autocorrelation by 0.01')\nplt.title('Train Sensor Time Auto-correlation')\nperiod = 25\nauto_corr_ks = list(range(period, max_lag, period))\nauto_corr = np.array([1] + [np.corrcoef(x[:-k], x[k:])[0, 1] for k in auto_corr_ks])\nplt.plot([0] + auto_corr_ks, auto_corr, 'go', alpha=0.5, label='strange autocorrelation at 0.25')\nperiod = 1675\nauto_corr_ks = list(range(period, max_lag, period))\nauto_corr = np.array([1] + [np.corrcoef(x[:-k], x[k:])[0, 1] for k in auto_corr_ks])\nplt.plot([0] + auto_corr_ks, auto_corr, 'ro', markersize=10, alpha=0.5, label='one week = 16.75?')\nplt.xlabel('k * 0.01 -  autocorrelation lag')\nplt.ylabel('autocorrelation')\nplt.legend(loc=0)\nfig.savefig('train_time_auto_correlation.png', dpi=300)","metadata":{"_cell_guid":"c29e43b4-a667-5ca8-3b43-39b3a83a9077"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The largest peaks are at approximately 1680 ticks.  Let's call it a week ;) \n\nIn each week we could see 7 local maxima ~ days.\n","metadata":{"_cell_guid":"dca1f4be-8077-f181-1e1e-3db2dd1a2d48"}},{"cell_type":"code","source":"week_duration = 1679\ntrain_time_cnt['week_part'] = ((train_time_cnt['time'].values * 100) % week_duration).astype(np.int64)\n# Aggregate weekly stats\ntrain_week_part = train_time_cnt.groupby(['week_part'])[['cnt']].sum().reset_index()\nfig = plt.figure()\nplt.plot(train_week_part.week_part.values, train_week_part.cnt.values, 'b.', alpha=0.5, label='train count')\ny_train = train_week_part['cnt'].rolling(window=20, center=True).mean().values\nplt.plot(train_week_part.week_part.values, y_train, 'b-', linewidth=4, alpha=0.5, label='train count smooth')\nplt.title('Relative Part of week')\nplt.ylabel('Number of records')\nplt.xlim(0, 1680)\nfig.savefig('week_duration.png', dpi=300)","metadata":{"_cell_guid":"3ee876be-a3d9-e141-eb21-0c361ce0a5a7"},"execution_count":null,"outputs":[]}]}