{"metadata": {"language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "nbconvert_exporter": "python", "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "version": "3.6.3", "pygments_lexer": "ipython3"}, "kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}}, "nbformat": 4, "cells": [{"metadata": {}, "source": ["This is an approach using ARIMA modelling, which is traditionally used to predict time series.\n", "\n", "Since auto.arima is only available in R, I did a function in python that exhaustively search for the best parameters of the ARIMA models"], "cell_type": "markdown"}, {"metadata": {"_uuid": "de8f7552cc7539c89fa6b73539bc1af1425b6fea", "_cell_guid": "d0a91d33-34c1-4b29-a659-4fc6021c2d3c"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import statsmodels.api as sm\n", "import statsmodels.tsa.api as smt\n", "import statsmodels.formula.api as smf\n", "\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "import warnings\n", "import itertools\n", "from sklearn.metrics import mean_squared_error\n", "import gc"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["#use smaller-size object types eg. int32 to make our dataframe more memory efficient\n", "types_dict = {'id': 'int32',\n", "             'item_nbr': 'int32',\n", "             'store_nbr': 'int8',\n", "             'unit_sales': 'float32'\n", "             }\n", "use_cols = [\"id\",\"date\",\"item_nbr\",\"store_nbr\",\"unit_sales\"]"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["#select dates from 10/7/2017\n", "grocery_train = pd.read_csv('../input/train.csv', low_memory=True,usecols=use_cols, dtype=types_dict,parse_dates=['date'],skiprows=range(1, 121688779))"]}, {"metadata": {}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_train.head()"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["use_cols2 = [\"id\",\"date\",\"item_nbr\",\"store_nbr\"]"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_test = pd.read_csv('../input/test.csv', low_memory=True,usecols=use_cols2, dtype=types_dict,parse_dates=['date'])"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["items_per_date_and_sales_grp=grocery_train.groupby(['item_nbr'])"]}, {"metadata": {}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["#total num of items in our data\n", "len(items_per_date_and_sales_grp)"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["#this function exhuastiely search for the optimal parameters(amount of AR, MA) and find the best ones with lowest AIC score.\n", "\n", "def gridSearch(itemObj,silent):\n", "    # Define the p, d and q parameters to take any value between 0 and 3\n", "    p = d = q = range(0, 3)\n", "\n", "    # Generate all different combinations of p, q and q triplets\n", "    pdq = list(itertools.product(p, d, q))\n", "\n", "    # Generate all different combinations of seasonal p, q and q triplets\n", "    seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]\n", "\n", "    bestAIC = np.inf\n", "    bestParam = None\n", "    bestSParam = None\n", "    \n", "    print('Running GridSearch')\n", "    \n", "    #use gridsearch to look for optimial arima parameters\n", "    for param in pdq:\n", "        for param_seasonal in seasonal_pdq:\n", "            try:\n", "                mod = sm.tsa.statespace.SARIMAX(itemObj,\n", "                                                order=param,\n", "                                                seasonal_order=param_seasonal,\n", "                                                enforce_stationarity=False,\n", "                                                enforce_invertibility=False)\n", "\n", "                results = mod.fit()\n", "\n", "                #if current run of AIC is better than the best one so far, overwrite it\n", "                if results.aic<bestAIC:\n", "                    bestAIC = results.aic\n", "                    bestParam = param\n", "                    bestSParam = param_seasonal\n", "\n", "            except:\n", "                continue\n", "                \n", "    print('the best ones are:',bestAIC,bestParam,bestSParam)\n", "    \n", "    print('proceeding to build a model with best parameter')\n", "    #apply the best parameters on the arima model\n", "    mod = sm.tsa.statespace.SARIMAX(itemObj,\n", "                                    order=bestParam,\n", "                                    seasonal_order=bestSParam,\n", "                                    enforce_stationarity=False,\n", "                                    enforce_invertibility=False)\n", "\n", "    results = mod.fit()\n", "\n", "    if(silent==False):\n", "        print(results.summary().tables[1])\n", "    \n", "    print(\"running diagnoistic plots\")\n", "    #results.plot_diagnostics(figsize=(15, 12))\n", "    #plt.show()\n", "    \n", "    #do a small test prediction\n", "    predictDateStr = '2017-08-01'\n", "    predictDate = pd.to_datetime(predictDateStr)\n", "    pred = results.get_prediction(start=predictDate,dynamic=True, full_results=True)\n", "    pred_ci = pred.conf_int()\n", "    \n", "    #calculting error scores\n", "    print(\"Validation mse:\", mean_squared_error(itemObj[predictDateStr:], pred.predicted_mean))\n", "    \n", "    if(silent==False):\n", "        #plot the prediction graph out\n", "        plt.plot(itemObj, color='black')\n", "        plt.plot(pred.predicted_mean,color='red', alpha=.7)\n", "\n", "        #ax = plt.gca()\n", "        #ax.fill_between(pred_ci.index,\n", "        #            pred_ci.iloc[:, 0],\n", "        #            pred_ci.iloc[:, 1], color='k', alpha=.15)\n", "        plt.show()\n", "    \n", "    #make forecast for next 16 days for submission data\n", "    n_steps = 16\n", "    pred_uc_99 = results.get_forecast(steps=n_steps, alpha=0.01) # alpha=0.01 signifies 99% confidence interval\n", "\n", "    # Get confidence intervals 95% & 99% of the forecasts\n", "    pred_ci_99 = pred_uc_99.conf_int()\n", "    \n", "    if(silent==False):\n", "        #plot forecase prediction\n", "        plt.plot(itemObj, color='black')\n", "        plt.plot(pred_uc_99.predicted_mean,color='red', alpha=.7)\n", "        ax = plt.gca()\n", "        ax.fill_between(pred_ci_99.index,\n", "                        pred_ci_99.iloc[:, 0],\n", "                        pred_ci_99.iloc[:, 1], color='k', alpha=.25)\n", "        plt.show()\n", "\n", "    print(pred_uc_99.predicted_mean)\n", "\n", "    #return forecasted result\n", "    return pred_uc_99.predicted_mean"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["#turn off warnings before running gridsearch\n", "import warnings\n", "warnings.filterwarnings(action='once')"]}, {"metadata": {}, "source": ["**The search for parameter below will take a long time to run, and likely to timeout in Kaggle's kernel. Use it offline at your own machine.**"], "cell_type": "markdown"}, {"metadata": {}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["listOfItems = []\n", "count = 0\n", "\n", "#go through each item group and put it through arima for prediction\n", "for name,grp in items_per_date_and_sales_grp:\n", "    count+=1\n", "    \n", "    print('run count',count)\n", "    #further group it by day, averaging the unit sales\n", "    itembyday = grp.groupby('date')['unit_sales'].mean()\n", "    \n", "    #make sure every item has valid data for our training date range(07-10 to 08-15)\n", "    date_index = pd.date_range('2017-07-10', '2017-08-15')\n", "    itembyday = itembyday.reindex(date_index,method='nearest')\n", "\n", "    #run modelling use extracted item day sales data\n", "    #show diagnosistic plots only for the first run\n", "    if count==1:\n", "        predictedVal = gridSearch(itembyday,False)\n", "    else:\n", "        predictedVal = gridSearch(itembyday,True)\n", "    \n", "    #create a dataframe from returned predicted values\n", "    predictedDF= pd.DataFrame(columns=['item_nbr','date','unit_sales'])\n", "    predictedDF['unit_sales'] = predictedDF['unit_sales'].astype('float32')\n", "    predictedDF['item_nbr'] = predictedDF['item_nbr'].astype('int32')\n", "\n", "    predictedDF['unit_sales']=predictedVal\n", "    predictedDF['date']=predictedVal.index\n", "    predictedDF['item_nbr']=name\n", "    \n", "    #append the dataframe into a list for later concat\n", "    listOfItems.append(predictedDF)\n", "    gc.collect()\n", "    #for testing n-th loop\n", "    #if count==3:\n", "    #    break"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["predDF = pd.concat(listOfItems)"]}, {"metadata": {}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["predDF"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_test = pd.merge(grocery_test,predDF,how='left',on=['item_nbr','date'])"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_test['unit_sales']=grocery_test['unit_sales'].clip(lower=0)"]}, {"metadata": {}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_test.shape"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_test = grocery_test.fillna(0)"]}, {"metadata": {}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["grocery_test[['id','unit_sales']].to_csv('grocery_submit.csv', index=False, float_format='%.3f')"]}, {"metadata": {"collapsed": true}, "outputs": [], "execution_count": null, "cell_type": "code", "source": []}], "nbformat_minor": 1}