{"metadata": {"language_info": {"pygments_lexer": "ipython3", "file_extension": ".py", "mimetype": "text/x-python", "version": "3.6.3", "nbconvert_exporter": "python", "name": "python", "codemirror_mode": {"version": 3, "name": "ipython"}}, "kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}}, "nbformat": 4, "nbformat_minor": 1, "cells": [{"cell_type": "markdown", "source": ["# Prepare"], "metadata": {"_cell_guid": "2887fbc8-496d-4033-b85d-02d545903f90", "_uuid": "6f2519b19ab9cfbbf515ed611d4a2438010bfc66"}}, {"cell_type": "code", "execution_count": null, "source": ["# -*- coding: utf-8 -*-\n", "import datetime\n", "from datetime import timedelta\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import tensorflow as tf\n", "\n", "from tensorflow.contrib.timeseries.python.timeseries import NumpyReader\n", "from tensorflow.contrib.timeseries.python.timeseries import estimators as tfts_estimators\n", "from tensorflow.contrib.timeseries.python.timeseries import model as tfts_model\n", "\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n"], "outputs": [], "metadata": {"_cell_guid": "791934a3-dfef-470e-93ba-5316e7b0f4cc", "_uuid": "fb6ed10e5af3dc13e8a610d268104d49f8b4c2e9"}}, {"cell_type": "markdown", "source": ["# Read Train data"], "metadata": {"_cell_guid": "3d621ab2-370a-4540-9b07-5355774c1533", "_uuid": "cbbd94a687b28f986dce3d613117c00e2095a806"}}, {"cell_type": "code", "execution_count": null, "source": ["dtypes = {'id':'int64', 'item_nbr':'int32', 'store_nbr':'int8'}\n", "\n", "train = pd.read_csv('../input/train.csv', usecols=[1,2,3,4], dtype=dtypes, parse_dates=['date'], \n", "                    skiprows=range(1, 101688780) #Skip initial dates\n", ")\n", "\n", "train.loc[(train.unit_sales < 0),'unit_sales'] = 0 # eliminate negatives\n", "train['unit_sales'] =  train['unit_sales'].apply(pd.np.log1p) #logarithm conversion\n", "train['dow'] = train['date'].dt.dayofweek "], "outputs": [], "metadata": {"_cell_guid": "d86b91d8-e8d2-4b62-862c-406079d14161", "_uuid": "dd4bf13cfa82f19be4093faad05216d78aea4b26"}}, {"cell_type": "code", "execution_count": null, "source": ["# creating records for all items, in all markets on all dates\n", "# for correct calculation of daily unit sales averages.\n", "u_dates = train.date.unique()\n", "u_stores = train.store_nbr.unique()\n", "u_items = train.item_nbr.unique()\n", "train.set_index(['date', 'store_nbr', 'item_nbr'], inplace=True)\n", "train = train.reindex(\n", "    pd.MultiIndex.from_product(\n", "        (u_dates, u_stores, u_items),\n", "        names=['date','store_nbr','item_nbr']\n", "    )\n", ")"], "outputs": [], "metadata": {"_cell_guid": "39576949-21f1-4740-80bb-073edc1a49ac", "_uuid": "2fdb26c33eff6454f9d2f8e6d228c957969bf624", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["train.loc[:, 'unit_sales'].fillna(0, inplace=True) # fill NaNs\n", "train.reset_index(inplace=True) # reset index and restoring unique columns  \n", "lastdate = train.iloc[train.shape[0]-1].date"], "outputs": [], "metadata": {"_cell_guid": "f07b4373-73d7-4421-a0bd-9937087000a0", "_uuid": "1811aa3be385237df174140b0b6f72313f5e4c3a", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["train.head()"], "outputs": [], "metadata": {"_cell_guid": "9e15054b-16f5-4ee2-997c-131763de4191", "_uuid": "98303cb457d329e701b46115c9f8d3f5d89c0719", "scrolled": true}}, {"cell_type": "code", "execution_count": null, "source": ["tmp = train[['item_nbr','store_nbr','dow','unit_sales']]\n", "ma_dw = tmp.groupby(['item_nbr','store_nbr','dow'])['unit_sales'].mean().to_frame('madw')\n", "ma_dw.reset_index(inplace=True)\n", "ma_dw.head()"], "outputs": [], "metadata": {"_cell_guid": "d6404245-80f8-44e9-9305-58c56e026fbf", "_uuid": "dd2dc6c5f6b1835b34e7f4711cc80fc5c9b9f8d2"}}, {"cell_type": "code", "execution_count": null, "source": ["tmp = ma_dw[['item_nbr','store_nbr','madw']]\n", "ma_wk = tmp.groupby(['item_nbr', 'store_nbr'])['madw'].mean().to_frame('mawk')\n", "ma_wk.reset_index(inplace=True)\n", "ma_wk.head()"], "outputs": [], "metadata": {"_cell_guid": "47ced6e6-a4aa-4c50-91be-00cc3a136d16", "_uuid": "1d09db10e17856079f746ec3cf8c90bced7bb4df"}}, {"cell_type": "markdown", "source": ["# Moving Average - Our Basic Model"], "metadata": {"_uuid": "e2dff589f16d6b44e7cebb68331e4b7f983e6815"}}, {"cell_type": "code", "execution_count": null, "source": ["tmp = train[['item_nbr','store_nbr','unit_sales']]\n", "ma_is = tmp.groupby(['item_nbr', 'store_nbr'])['unit_sales'].mean().to_frame('mais226')"], "outputs": [], "metadata": {"_uuid": "dbadfeb78b48f8dfb98546f0dba24788b22c2cfe", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["for i in [112,56,28,14,7,3,1]:\n", "    tmp = train[train.date>lastdate-timedelta(int(i))]\n", "    tmpg = tmp.groupby(['item_nbr','store_nbr'])['unit_sales'].mean().to_frame('mais'+str(i))\n", "    ma_is = ma_is.join(tmpg, how='left')\n", "\n", "del tmp,tmpg"], "outputs": [], "metadata": {"_uuid": "3de8cfb1926c7f9a627e16de48377e2c6a93ea7f", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["ma_is['mais']=ma_is.median(axis=1)\n", "ma_is.reset_index(inplace=True)"], "outputs": [], "metadata": {"_uuid": "22c535688c57e0bfa8e25b0fc2cadf8c3287bd85", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["ma_is.head()"], "outputs": [], "metadata": {}}, {"cell_type": "markdown", "source": ["# Tensorflow Timesereies - ARRegressor"], "metadata": {"_cell_guid": "ee38b647-ea41-4353-b26b-7c0ad5da9ff0", "_uuid": "9c8772b5d69b06666c0e609ccbcabdad3f3db44a"}}, {"cell_type": "code", "execution_count": null, "source": ["def data_to_npreader(store_nbr: int, item_nbr: int) -> NumpyReader:\n", "    unit_sales = train[np.logical_and(train[\"store_nbr\"] == store_nbr,\n", "                                      train['item_nbr'] == item_nbr)].unit_sales\n", "\n", "    x = np.asarray(range(len(unit_sales)))\n", "    y = np.asarray(unit_sales)\n", "\n", "    dataset = {\n", "        tf.contrib.timeseries.TrainEvalFeatures.TIMES: x,\n", "        tf.contrib.timeseries.TrainEvalFeatures.VALUES: y,\n", "    }\n", "\n", "    reader = NumpyReader(dataset)\n", "    return x, y, reader"], "outputs": [], "metadata": {"collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["x, y, reader = data_to_npreader(store_nbr=1, item_nbr=105574)\n", "\n", "train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(\n", "        reader, batch_size=32, window_size=40)\n", "\n", "ar = tf.contrib.timeseries.ARRegressor(\n", "    periodicities=21, input_window_size=30, output_window_size=10,\n", "    num_features=1,\n", "    loss=tf.contrib.timeseries.ARModel.NORMAL_LIKELIHOOD_LOSS\n", ")\n", "\n", "ar.train(input_fn=train_input_fn, steps=16000)"], "outputs": [], "metadata": {}}, {"cell_type": "code", "execution_count": null, "source": ["evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)\n", "# keys of evaluation: ['covariance', 'loss', 'mean', 'observed', 'start_tuple', 'times', 'global_step']\n", "evaluation = ar.evaluate(input_fn=evaluation_input_fn, steps=1)\n", "\n", "(ar_predictions,) = tuple(ar.predict(\n", "    input_fn=tf.contrib.timeseries.predict_continuation_input_fn(\n", "        evaluation, steps=16)))"], "outputs": [], "metadata": {"_cell_guid": "1cdc09b0-88be-494c-bc0b-ee266db7b568", "_uuid": "9f737adbef3da70ca4a0e395d443c72b720448f5"}}, {"cell_type": "code", "execution_count": null, "source": ["plt.figure(figsize=(15, 5))\n", "plt.plot(x.reshape(-1), y.reshape(-1), label='origin')\n", "plt.plot(evaluation['times'].reshape(-1), evaluation['mean'].reshape(-1), label='evaluation')\n", "plt.plot(ar_predictions['times'].reshape(-1), ar_predictions['mean'].reshape(-1), label='prediction')\n", "plt.xlabel('time_step')\n", "plt.ylabel('values')\n", "plt.legend(loc=4)\n", "plt.show()"], "outputs": [], "metadata": {"_cell_guid": "95b6f2c8-f005-41d7-bddb-14ce7f7a1705", "_uuid": "5bee10b5b487fcc904582dd79be811c1278a4ca2"}}, {"cell_type": "markdown", "source": ["# Tensorflow Timesereies - LSTM"], "metadata": {}}, {"cell_type": "code", "execution_count": null, "source": ["class _LSTMModel(tfts_model.SequentialTimeSeriesModel):\n", "    \"\"\"A time series model-building example using an RNNCell.\"\"\"\n", "    \n", "    def __init__(self, num_units, num_features, dtype=tf.float32):\n", "        \"\"\"Initialize/configure the model object.\n", "        Note that we do not start graph building here. Rather, this object is a\n", "        configurable factory for TensorFlow graphs which are run by an Estimator.\n", "        Args:\n", "          num_units: The number of units in the model's LSTMCell.\n", "          num_features: The dimensionality of the time series (features per\n", "            timestep).\n", "          dtype: The floating point data type to use.\n", "        \"\"\"\n", "        \n", "        super(_LSTMModel, self).__init__(\n", "            # Pre-register the metrics we'll be outputting (just a mean here).\n", "            train_output_names=[\"mean\"],\n", "            predict_output_names=[\"mean\"],\n", "            num_features=num_features,\n", "            dtype=dtype)\n", "        self._num_units = num_units\n", "        # Filled in by initialize_graph()\n", "        self._lstm_cell = None\n", "        self._lstm_cell_run = None\n", "        self._predict_from_lstm_output = None\n", "\n", "    def initialize_graph(self, input_statistics):\n", "        \"\"\"Save templates for components, which can then be used repeatedly.\n", "        This method is called every time a new graph is created. It's safe to start\n", "        adding ops to the current default graph here, but the graph should be\n", "        constructed from scratch.\n", "        Args:\n", "          input_statistics: A math_utils.InputStatistics object.\n", "        \"\"\"\n", "        \n", "        super(_LSTMModel, self).initialize_graph(input_statistics=input_statistics)\n", "        self._lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=self._num_units)\n", "        # Create templates so we don't have to worry about variable reuse.\n", "        self._lstm_cell_run = tf.make_template(\n", "            name_=\"lstm_cell\",\n", "            func_=self._lstm_cell,\n", "            create_scope_now_=True)\n", "        # Transforms LSTM output into mean predictions.\n", "        self._predict_from_lstm_output = tf.make_template(\n", "            name_=\"predict_from_lstm_output\",\n", "            func_=\n", "            lambda inputs: tf.layers.dense(inputs=inputs, units=self.num_features),\n", "            create_scope_now_=True)\n", "\n", "    def get_start_state(self):\n", "        \"\"\"Return initial state for the time series model.\"\"\"\n", "        return (\n", "            # Keeps track of the time associated with this state for error checking.\n", "            tf.zeros([], dtype=tf.int64),\n", "            # The previous observation or prediction.\n", "            tf.zeros([self.num_features], dtype=self.dtype),\n", "            # The state of the RNNCell (batch dimension removed since this parent\n", "            # class will broadcast).\n", "            [tf.squeeze(state_element, axis=0)\n", "             for state_element\n", "             in self._lstm_cell.zero_state(batch_size=1, dtype=self.dtype)])\n", "\n", "    def _filtering_step(self, current_times, current_values, state, predictions):\n", "        \"\"\"Update model state based on observations.\n", "        Note that we don't do much here aside from computing a loss. In this case\n", "        it's easier to update the RNN state in _prediction_step, since that covers\n", "        running the RNN both on observations (from this method) and our own\n", "        predictions. This distinction can be important for probabilistic models,\n", "        where repeatedly predicting without filtering should lead to low-confidence\n", "        predictions.\n", "        Args:\n", "          current_times: A [batch size] integer Tensor.\n", "          current_values: A [batch size, self.num_features] floating point Tensor\n", "            with new observations.\n", "          state: The model's state tuple.\n", "          predictions: The output of the previous `_prediction_step`.\n", "        Returns:\n", "          A tuple of new state and a predictions dictionary updated to include a\n", "          loss (note that we could also return other measures of goodness of fit,\n", "          although only \"loss\" will be optimized).\n", "        \"\"\"\n", "        state_from_time, prediction, lstm_state = state\n", "        with tf.control_dependencies(\n", "            [tf.assert_equal(current_times, state_from_time)]):\n", "          # Subtract the mean and divide by the variance of the series.  Slightly\n", "          # more efficient if done for a whole window (using the normalize_features\n", "          # argument to SequentialTimeSeriesModel).\n", "          transformed_values = self._scale_data(current_values)\n", "          # Use mean squared error across features for the loss.\n", "          predictions[\"loss\"] = tf.reduce_mean(\n", "              (prediction - transformed_values) ** 2, axis=-1)\n", "          # Keep track of the new observation in model state. It won't be run\n", "          # through the LSTM until the next _imputation_step.\n", "          new_state_tuple = (current_times, transformed_values, lstm_state)\n", "        return (new_state_tuple, predictions)\n", "\n", "    def _prediction_step(self, current_times, state):\n", "        \"\"\"Advance the RNN state using a previous observation or prediction.\"\"\"\n", "        _, previous_observation_or_prediction, lstm_state = state\n", "        lstm_output, new_lstm_state = self._lstm_cell_run(\n", "            inputs=previous_observation_or_prediction, state=lstm_state)\n", "        next_prediction = self._predict_from_lstm_output(lstm_output)\n", "        new_state_tuple = (current_times, next_prediction, new_lstm_state)\n", "        return new_state_tuple, {\"mean\": self._scale_back_data(next_prediction)}\n", "\n", "    def _imputation_step(self, current_times, state):\n", "        \"\"\"Advance model state across a gap.\"\"\"\n", "        # Does not do anything special if we're jumping across a gap. More advanced\n", "        # models, especially probabilistic ones, would want a special case that\n", "        # depends on the gap size.\n", "        return state\n", "\n", "    def _exogenous_input_step(\n", "        self, current_times, current_exogenous_regressors, state):\n", "        \"\"\"Update model state based on exogenous regressors.\"\"\"\n", "        raise NotImplementedError(\n", "            \"Exogenous inputs are not implemented for this example.\")"], "outputs": [], "metadata": {"collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["x, y, reader = data_to_npreader(store_nbr=2, item_nbr=105574)\n", "\n", "train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(\n", "      reader, batch_size=16, window_size=21)\n", "\n", "estimator = tfts_estimators.TimeSeriesRegressor(\n", "      model=_LSTMModel(num_features=1, num_units=32),\n", "      optimizer=tf.train.AdamOptimizer(0.001))\n", "\n", "estimator.train(input_fn=train_input_fn, steps=16000)\n", "evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)\n", "evaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)"], "outputs": [], "metadata": {}}, {"cell_type": "code", "execution_count": null, "source": ["(lstm_predictions,) = tuple(estimator.predict(\n", "      input_fn=tf.contrib.timeseries.predict_continuation_input_fn(\n", "          evaluation, steps=16)))"], "outputs": [], "metadata": {}}, {"cell_type": "code", "execution_count": null, "source": ["plt.figure(figsize=(15, 5))\n", "plt.plot(x.reshape(-1), y.reshape(-1), label='origin')\n", "plt.plot(evaluation['times'].reshape(-1), evaluation['mean'].reshape(-1), label='evaluation')\n", "plt.plot(lstm_predictions['times'].reshape(-1), lstm_predictions['mean'].reshape(-1), label='prediction')\n", "plt.xlabel('time_step')\n", "plt.ylabel('values')\n", "plt.legend(loc=4)\n", "plt.show()"], "outputs": [], "metadata": {}}, {"cell_type": "markdown", "source": ["# Forecasting Test data"], "metadata": {"_cell_guid": "3e64d9f4-5789-4ca9-a31f-2307b4528b66", "_uuid": "5364f70e08646a689374e90b6f5c8f94d712b5b7"}}, {"cell_type": "code", "execution_count": null, "source": ["# Read test dataset\n", "test = pd.read_csv('../input/test.csv', dtype=dtypes, parse_dates=['date'])\n", "test['dow'] = test['date'].dt.dayofweek\n", "\n", "# Moving Average\n", "test = pd.merge(test, ma_is, how='left', on=['item_nbr','store_nbr'])\n", "test = pd.merge(test, ma_wk, how='left', on=['item_nbr','store_nbr'])\n", "test = pd.merge(test, ma_dw, how='left', on=['item_nbr','store_nbr','dow'])\n", "test['unit_sales'] = test.mais\n", "\n", "# Autoregressive\n", "ar_predictions['mean'][ar_predictions['mean'] < 0] = 0\n", "test.loc[np.logical_and(test['store_nbr'] == 1, test['item_nbr'] == 105574), 'unit_sales'] = ar_predictions['mean']\n", "\n", "# LSTM\n", "lstm_predictions['mean'][lstm_predictions['mean'] < 0] = 0\n", "test.loc[np.logical_and(test['store_nbr'] == 2, test['item_nbr'] == 105574), 'unit_sales'] = lstm_predictions['mean']"], "outputs": [], "metadata": {"_cell_guid": "67df9b2d-499a-4a73-b02a-32f72aa07cd4", "_uuid": "19a8725a462fa90c7b98a1b9075d19b5e2e72d7c", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["pos_idx = test['mawk'] > 0\n", "test_pos = test.loc[pos_idx]\n", "test.loc[pos_idx, 'unit_sales'] = test_pos['unit_sales'] * test_pos['madw'] / test_pos['mawk']"], "outputs": [], "metadata": {"_cell_guid": "ee48e38c-aef9-4131-9498-689636a0f665", "_uuid": "4f5f0edbbcff0e2ba99f0c07a6dd001cff516ed2", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["test.loc[:, \"unit_sales\"].fillna(0, inplace=True)\n", "test['unit_sales'] = test['unit_sales'].apply(pd.np.expm1) # restoring unit values \n", "test['mais'] = test['mais'].apply(pd.np.expm1) # restoring unit values "], "outputs": [], "metadata": {"_cell_guid": "f82e2a4f-8b01-4a05-a695-fb6ec3cc5447", "_uuid": "2347f4b4dc4f50b20ee073143e77e3b4f27a4d08", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["holiday = pd.read_csv('../input/holidays_events.csv', parse_dates=['date'])\n", "holiday = holiday.loc[holiday['transferred'] == False]\n", "\n", "test = pd.merge(test, holiday, how = 'left', on =['date'] )\n", "test['transferred'].fillna(True, inplace=True)\n", "\n", "test.loc[test['transferred'] == False, 'unit_sales'] *= 1.2\n", "test.loc[test['onpromotion'] == True, 'unit_sales'] *= 1.5"], "outputs": [], "metadata": {"_uuid": "087eab72f32ac3c0bec3f6b82bbeb61d456868da", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "source": ["test.loc[np.logical_and(test['store_nbr'] == 1, test['item_nbr'] == 105574)]"], "outputs": [], "metadata": {}}, {"cell_type": "code", "execution_count": null, "source": ["test[['id','unit_sales']].to_csv('submission.csv.gz', index=False, compression='gzip')"], "outputs": [], "metadata": {"_uuid": "671d5646b0defc79a1ad0272a4714347772ffb7b", "collapsed": true}}]}