{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Install Packages to Extract Data","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n!pip install pyunpack\n!pip install patool\nos.system('apt-get install p7zip')\n\n\nfrom pyunpack import Archive\nimport shutil\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:23:01.464130Z","iopub.execute_input":"2021-06-28T03:23:01.464747Z","iopub.status.idle":"2021-06-28T03:23:17.446351Z","shell.execute_reply.started":"2021-06-28T03:23:01.464662Z","shell.execute_reply":"2021-06-28T03:23:17.445199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Dir to store data","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n#Archive('/kaggle/input/favorita-grocery-sales-forecasting/').extractall('/kaggle/working/')\nfor dirname, _, filenames in os.walk('/kaggle/input/'):\n    for filename in filenames:\n        if not os.path.exists('/kaggle/working/'+filename.split(\".\")[0]+\"/\"):\n            os.makedirs('/kaggle/working/'+filename.split(\".\")[0]+\"/\")\n        #print(os.path.join(dirname, filename))\n#        if not os.path.exists('/kaggle/working/'+filename.split(\".\")[0]+\"/\"):\n            #Archive(os.path.join(dirname, filename)).extractall('/kaggle/working/'+filename.split(\".\")[0]+\"/\")\n        #print(os.path.join(dirname, filename))\n        #print(filename.split(\".\")[0])\n\n      \n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:23:17.449920Z","iopub.execute_input":"2021-06-28T03:23:17.450204Z","iopub.status.idle":"2021-06-28T03:23:17.456229Z","shell.execute_reply.started":"2021-06-28T03:23:17.450173Z","shell.execute_reply":"2021-06-28T03:23:17.455451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract All Data","metadata":{}},{"cell_type":"code","source":"Archive(\"/kaggle/input/favorita-grocery-sales-forecasting/train.csv.7z\").extractall('/kaggle/working/train')\nArchive(\"/kaggle/input/favorita-grocery-sales-forecasting/test.csv.7z\").extractall('/kaggle/working/test')\nArchive(\"/kaggle/input/favorita-grocery-sales-forecasting/oil.csv.7z\").extractall('/kaggle/working/oil')\nArchive(\"/kaggle/input/favorita-grocery-sales-forecasting/stores.csv.7z\").extractall('/kaggle/working/stores')\nArchive(\"/kaggle/input/favorita-grocery-sales-forecasting/items.csv.7z\").extractall('/kaggle/working/items')\nArchive(\"/kaggle/input/favorita-grocery-sales-forecasting/transactions.csv.7z\").extractall('/kaggle/working/transactions')\nArchive(\"/kaggle/input/favorita-grocery-sales-forecasting/holidays_events.csv.7z\").extractall('/kaggle/working/holidays_events')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:23:17.457773Z","iopub.execute_input":"2021-06-28T03:23:17.458035Z","iopub.status.idle":"2021-06-28T03:24:24.360780Z","shell.execute_reply.started":"2021-06-28T03:23:17.458009Z","shell.execute_reply":"2021-06-28T03:24:24.359432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation\n1. Load Data\n2. Reduce Memory\n3. Merge Data","metadata":{}},{"cell_type":"code","source":"\ndtypes = {'id':'uint32', 'item_nbr':'int32', 'store_nbr':'int8', 'onpromotion':str}\ndata = {\n    'tra': pd.read_csv('../working/train/train.csv', dtype=dtypes, parse_dates=['date']),\n    'tes': pd.read_csv('../working/test/test.csv', dtype=dtypes, parse_dates=['date']),\n    'ite': pd.read_csv('../working/items/items.csv'),\n    'sto': pd.read_csv('../working/stores/stores.csv'),\n    'trn': pd.read_csv('../working/transactions/transactions.csv', parse_dates=['date']),\n    'hol': pd.read_csv('../working/holidays_events/holidays_events.csv', dtype={'transferred':str}, parse_dates=['date']),\n    'oil': pd.read_csv('../working/oil/oil.csv', parse_dates=['date']),\n    }","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:24:24.362810Z","iopub.execute_input":"2021-06-28T03:24:24.363203Z","iopub.status.idle":"2021-06-28T03:25:35.453567Z","shell.execute_reply.started":"2021-06-28T03:24:24.363138Z","shell.execute_reply":"2021-06-28T03:25:35.452646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#start_mem = test.memory_usage().sum() / 1024**2\n#print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:25:35.454771Z","iopub.execute_input":"2021-06-28T03:25:35.455042Z","iopub.status.idle":"2021-06-28T03:25:35.458240Z","shell.execute_reply.started":"2021-06-28T03:25:35.455015Z","shell.execute_reply":"2021-06-28T03:25:35.457232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Top 10 selling products across geographis [In termrs of Daily Transaction] \n\nDeprecated: Item Number 329362 in Store Number 44","metadata":{}},{"cell_type":"code","source":"#test = test.merge(right = data['ite'], on='item_nbr')\n#test = test.merge(right=data['sto'],on='store_nbr')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:25:35.479685Z","iopub.execute_input":"2021-06-28T03:25:35.479946Z","iopub.status.idle":"2021-06-28T03:25:35.488350Z","shell.execute_reply.started":"2021-06-28T03:25:35.479920Z","shell.execute_reply":"2021-06-28T03:25:35.487315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(15,6))\ntrain = data['tra']\ntop_ten_selling_products = train.groupby(['item_nbr']).count()['id'].sort_values(ascending=False).head(10)\nax = top_ten_selling_products.plot.bar(x='item_nbr', y='id', rot=0)\nplt.ylabel(\"Total Unit Sold\")\nplt.title(\"Top 10 Selling Products\")\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:42:54.365985Z","iopub.execute_input":"2021-06-28T03:42:54.366296Z","iopub.status.idle":"2021-06-28T03:43:06.730735Z","shell.execute_reply.started":"2021-06-28T03:42:54.366269Z","shell.execute_reply":"2021-06-28T03:43:06.729748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 10 Selling Product Counts","metadata":{}},{"cell_type":"code","source":"top_ten_selling_products.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:25:49.457662Z","iopub.execute_input":"2021-06-28T03:25:49.458198Z","iopub.status.idle":"2021-06-28T03:25:49.465396Z","shell.execute_reply.started":"2021-06-28T03:25:49.458147Z","shell.execute_reply":"2021-06-28T03:25:49.464651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Daily Trasaction Plot of top 10 Selling products\n\nAlso Save files of each product as a time series in directory\n","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16,8))\n#i=1\nfor item_nbr in top_ten_selling_products.index:\n    top1 = data['tra'][data['tra'].item_nbr == item_nbr]\n    #filename = str(i)+' '+str(item_nbr)+'.csv'\n    #top1.to_csv(filename, index=False)\n    #i=i+1\n    top1.groupby(['date']).sum()['unit_sales'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0, label=item_nbr)\n    plt.legend(loc=\"upper right\")\nplt.ylabel(\"Total Unit Sold\")\nplt.title(\"Top 10 Selling Products\")\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:45:15.481029Z","iopub.execute_input":"2021-06-28T03:45:15.481370Z","iopub.status.idle":"2021-06-28T03:45:17.430524Z","shell.execute_reply.started":"2021-06-28T03:45:15.481343Z","shell.execute_reply":"2021-06-28T03:45:17.429535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## There are three extream events \n\n1. [First is  between 2013 and 2014]\n\n2. [Second is between 2015 and 2016] \n\n3. [Second is between 2016 and 2017] \n\nOut objective is to estimate extream event in 2016-17\n","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,6))\ntop1 = data['tra'][data['tra'].item_nbr == 265559 ]\n#top1.groupby(['date']).sum()['unit_sales'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0, label=265559, color=\"yellow\")\n#plt.plot()\ntop1.groupby(['date']).sum()['unit_sales']['2013'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0,label = 2013, color='green')\nplt.plot()\ntop1.groupby(['date']).sum()['unit_sales']['2014'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0,label = 2014, color='blue')\nplt.plot()\ntop1.groupby(['date']).sum()['unit_sales']['2015'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0,label = 2015, color='red')\nplt.plot()\ntop1.groupby(['date']).sum()['unit_sales']['2016'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0,label = 2016, color='orange')\nplt.plot()\ntop1.groupby(['date']).sum()['unit_sales']['2017'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0,label = 2017, color='brown')\nplt.plot()\n\nplt.ylabel(\"Total Unit Sold\")\nplt.title(\"Year Wise Products Sale\")\nplt.legend(loc=\"upper right\")","metadata":{"execution":{"iopub.status.busy":"2021-06-28T05:25:47.401233Z","iopub.execute_input":"2021-06-28T05:25:47.401927Z","iopub.status.idle":"2021-06-28T05:25:48.605189Z","shell.execute_reply.started":"2021-06-28T05:25:47.401877Z","shell.execute_reply":"2021-06-28T05:25:48.603955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using statistical method for estimation","metadata":{}},{"cell_type":"markdown","source":"#### Prepare time Series for estimation","metadata":{}},{"cell_type":"code","source":"unit_sales_by_date = top1.groupby('date').sum()['unit_sales']","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:25:52.147045Z","iopub.execute_input":"2021-06-28T03:25:52.147429Z","iopub.status.idle":"2021-06-28T03:25:52.163530Z","shell.execute_reply.started":"2021-06-28T03:25:52.147387Z","shell.execute_reply":"2021-06-28T03:25:52.162731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import Evaluation Metrics","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_percentage_error","metadata":{"execution":{"iopub.status.busy":"2021-06-28T03:25:52.164927Z","iopub.execute_input":"2021-06-28T03:25:52.165294Z","iopub.status.idle":"2021-06-28T03:25:53.140300Z","shell.execute_reply.started":"2021-06-28T03:25:52.165255Z","shell.execute_reply":"2021-06-28T03:25:53.139367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Adfuller Test to check stationary time series","metadata":{}},{"cell_type":"code","source":"from statsmodels.tsa.stattools import adfuller\nX = unit_sales_by_date.values\nresult = adfuller(X)\nprint('ADF Statistic: %f' % result[0])\nprint('p-value: %f' % result[1])\nprint('Critical Values:')\nfor key, value in result[4].items():\n    print('\\t%s: %.3f' % (key, value))\n\nif result[0] < result[4][\"5%\"]:\n    print (\"Reject Ho - Time Series is Stationary\")\nelse:\n    print (\"Failed to Reject Ho - Time Series is Non-Stationary\")","metadata":{"execution":{"iopub.status.busy":"2021-06-28T05:30:05.720605Z","iopub.execute_input":"2021-06-28T05:30:05.721240Z","iopub.status.idle":"2021-06-28T05:30:05.794239Z","shell.execute_reply.started":"2021-06-28T05:30:05.721203Z","shell.execute_reply":"2021-06-28T05:30:05.793132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check ACF and PACF to decide model order","metadata":{}},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_pacf\nfrom statsmodels.graphics.tsaplots import plot_acf\nplot_pacf(unit_sales_by_date)\nplot_acf(unit_sales_by_date, lags=7)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T05:27:48.831471Z","iopub.execute_input":"2021-06-28T05:27:48.831825Z","iopub.status.idle":"2021-06-28T05:27:49.270604Z","shell.execute_reply.started":"2021-06-28T05:27:48.831795Z","shell.execute_reply":"2021-06-28T05:27:49.269659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Moving Average Model","metadata":{}},{"cell_type":"code","source":"def moving_average_forecast(series, window_size):\n    \"\"\"Forecasts the mean of the last few values.\n     If window_size=1, then this is equivalent to naive forecast\"\"\"\n    forecast = []\n    for time in range(len(series) - window_size):\n        forecast.append(series[time:time + window_size].mean())\n    return np.array(forecast)\nsplit_time = 1455\nmoving_average_days = 6\nmoving_avg = moving_average_forecast(unit_sales_by_date,moving_average_days )# [split_time + 7:]\n\nprint(moving_avg.shape,unit_sales_by_date.shape)\n\nprint(\"mean_squared_error\",mean_squared_error(unit_sales_by_date.values[moving_average_days:], moving_avg))\nprint(\"mean_absolute_error\",mean_absolute_error(unit_sales_by_date.values[moving_average_days:], moving_avg))\nprint(\"mean_absolute_percentage_error\",mean_absolute_percentage_error(unit_sales_by_date.values[moving_average_days:], moving_avg))\n\n# Plot Moving average vs Actual\nplt.figure(figsize=(15,6))\n\nplt.plot(unit_sales_by_date.values[moving_average_days:], label=\"Actual\")\n#plt.plot(moving_avg[1455:], label=\"Test Data\", color=\"Red\")\nplt.plot(moving_avg, label=\"Moving average\")\nplt.ylabel(\"Total Unit Sold\")\nplt.xlabel(\"Day\")\nplt.title(\"Moving Average Forecast\")\nplt.legend(loc=\"upper right\")","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:02:23.113283Z","iopub.execute_input":"2021-06-28T04:02:23.113662Z","iopub.status.idle":"2021-06-28T04:02:23.564042Z","shell.execute_reply.started":"2021-06-28T04:02:23.113626Z","shell.execute_reply":"2021-06-28T04:02:23.562970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## AR Model ","metadata":{}},{"cell_type":"code","source":"x_train = unit_sales_by_date['2013':'2016'].values\nx_test  = unit_sales_by_date['2017'].values\n\ndf = pd.DataFrame()\n\ndf[\"Original Values\"]  = unit_sales_by_date\ndf[\"shift1\"] = df[\"Original Values\"].shift()\ndf[\"shift2\"] = df[\"shift1\"].shift()\ndf[\"shift3\"] = df[\"shift2\"].shift()\ndf[\"shift4\"] = df[\"shift3\"].shift()\ndf[\"shift5\"] = df[\"shift4\"].shift()\ndf[\"shift6\"] = df[\"shift5\"].shift()\n#df[\"shift7\"] = df[\"shift6\"].shift()\n#df[\"shift8\"] = df[\"shift7\"].shift()\nlag_value = 6\ndf.dropna(inplace=True)\n#df.head(10)\n\nx_train, y_train = df['2013':'2016'].iloc[:,0:lag_value].values, df['2013':'2016'].iloc[:,lag_value:].values\nx_test, y_test = df['2017'].iloc[:,0:lag_value].values, df['2017'].iloc[:,lag_value:].values\n\nfrom sklearn.linear_model import LinearRegression\nreg = LinearRegression().fit(x_train, y_train)\nprint(reg.coef_)\nprint(reg.intercept_)\n\n\nar_predictions = reg.predict(x_test)\n\nplt.figure(figsize=(15,6))\nplt.plot(ar_predictions ,label = \"Predictions\")\nplt.plot(y_test, label = \"Original\" )\nplt.xlabel(\"Days\")\nplt.ylabel(\"Sale Count\")\n\nprint(\"mean_squared_error\",mean_squared_error(y_test, ar_predictions))\nprint(\"mean_absolute_error\",mean_absolute_error(y_test, ar_predictions))\nprint(\"mean_absolute_percentage_error\",mean_absolute_percentage_error(y_test, ar_predictions))\n\n#ar_predictions = reg.predict(x_train)\n#plt.figure(figsize=(15,6))\n#plt.plot(ar_predictions,label = \"Predictions\")\n#plt.plot(y_test, label = \"Y_train\")\nplt.legend(loc=\"upper right\")\n#print(\"mean_squared_error\",mean_squared_error(y_test, ar_predictions))\n#print(\"mean_absolute_error\",mean_absolute_error(y_test, ar_predictions))\n#print(\"mean_absolute_percentage_error\",mean_absolute_percentage_error(y_test, ar_predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:14:30.639175Z","iopub.execute_input":"2021-06-28T04:14:30.639564Z","iopub.status.idle":"2021-06-28T04:14:30.830264Z","shell.execute_reply.started":"2021-06-28T04:14:30.639513Z","shell.execute_reply":"2021-06-28T04:14:30.829539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ARMA Model","metadata":{}},{"cell_type":"code","source":"import statsmodels.api as sm\nfrom statsmodels.tsa.arima_model import ARMA\nfrom datetime import datetime, timedelta\n\nx_train, y_train = df['2013':'2016'].iloc[:,0:lag_value], df['2013':'2016'].iloc[:,lag_value:]\nx_test, y_test = df['2017'].iloc[:,0:lag_value], df['2017'].iloc[:,lag_value:]\n\nARMA_model = ARMA(x_train.values.reshape(-1).tolist(), (6,6)).fit()\narma_predictions = ARMA_model.predict(start=len(x_train), end=len(x_train) + len(x_test)-1, dynamic=False)\n\n\nplt.figure(figsize=(15,6))\nplt.plot(arma_predictions ,label = \"Predictions\")\nplt.plot(y_test.values, label = \"Original\" )\nplt.xlabel(\"Days\")\nplt.ylabel(\"Sale Count\")\nplt.legend(loc=\"upper right\")\n\nprint(\"mean_squared_error\",mean_squared_error(y_test, arma_predictions))\nprint(\"mean_absolute_error\",mean_absolute_error(y_test, arma_predictions))\nprint(\"mean_absolute_percentage_error\",mean_absolute_percentage_error(y_test, arma_predictions))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:48:09.381424Z","iopub.execute_input":"2021-06-28T04:48:09.382001Z","iopub.status.idle":"2021-06-28T04:48:22.740364Z","shell.execute_reply.started":"2021-06-28T04:48:09.381963Z","shell.execute_reply":"2021-06-28T04:48:22.739646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprediction_rolling = unit_sales_by_date\nfor end_date in x_test.index:\n    train_data = unit_sales_by_date[:end_date-timedelta(days=1)]\n    print(end_date,train_data.shape)\n    model = ARMA(train_data, (6,6)).fit()\n\n    \n    #model_fit = model.fit()\n    #pred = model_fit.predict(end_data)\n    #prediction_rolling.loc[end_date] = pred.loc[end_date]\n#prediction_rolling.head()\n'''\n#print(\"mean_squared_error\",mean_squared_error(x_test, predictions))\n#print(\"mean_absolute_error\",mean_absolute_error(x_test, predictions))\n#print(\"mean_absolute_percentage_error\",mean_absolute_percentage_error(x_test, predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-28T04:53:16.128316Z","iopub.execute_input":"2021-06-28T04:53:16.128688Z","iopub.status.idle":"2021-06-28T04:53:16.135131Z","shell.execute_reply.started":"2021-06-28T04:53:16.128656Z","shell.execute_reply":"2021-06-28T04:53:16.134122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ARIMA model","metadata":{}},{"cell_type":"code","source":"from statsmodels.tsa.arima_model import ARIMA\n\nmodel = ARIMA(x_train.values.reshape(-1).tolist(), order=(6,2,6)).fit()\n\narima_predictions = []\n\narima_predictions = model.predict(start=len(x_train),end=len(x_train) + len(x_test)-1,dynamic=False)\n#evaluate(predictions, x_test)\n\nprint(\"mean_squared_error\",mean_squared_error(y_test, arima_predictions))\nprint(\"mean_absolute_error\",mean_absolute_error(y_test, arima_predictions))\nprint(\"mean_absolute_percentage_error\",mean_absolute_percentage_error(y_test, arima_predictions))\n\nplt.figure(figsize=(15,6))\nplt.plot(arima_predictions ,label = \"Predictions\")\nplt.plot(y_test.values, label = \"Original\" )\nplt.xlabel(\"Days\")\nplt.ylabel(\"Sale Count\")\nplt.legend(loc=\"upper right\")","metadata":{"execution":{"iopub.status.busy":"2021-06-28T05:08:04.577583Z","iopub.execute_input":"2021-06-28T05:08:04.577974Z","iopub.status.idle":"2021-06-28T05:08:26.468088Z","shell.execute_reply.started":"2021-06-28T05:08:04.577942Z","shell.execute_reply":"2021-06-28T05:08:26.466954Z"},"trusted":true},"execution_count":null,"outputs":[]}]}