{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"},{"sourceId":7100531,"sourceType":"datasetVersion","datasetId":4093056}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom joblib import dump, load\nfrom datetime import datetime\n\nimport os\n\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom sklearn.linear_model import PassiveAggressiveRegressor\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        filepath = os.path.join(dirname, filename)\n        print(f'extracting to /kaggle/working/ ... {filepath}')\n        !p7zip -d -f -k {filepath}\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-03T22:29:43.861120Z","iopub.execute_input":"2023-12-03T22:29:43.861675Z","iopub.status.idle":"2023-12-03T22:31:13.189128Z","shell.execute_reply.started":"2023-12-03T22:29:43.861623Z","shell.execute_reply":"2023-12-03T22:31:13.187669Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{"execution":{"iopub.status.busy":"2023-12-01T16:23:46.320121Z","iopub.execute_input":"2023-12-01T16:23:46.320484Z","iopub.status.idle":"2023-12-01T16:23:46.502599Z","shell.execute_reply.started":"2023-12-01T16:23:46.320457Z","shell.execute_reply":"2023-12-01T16:23:46.500848Z"},"jupyter":{"source_hidden":true}}},{"cell_type":"code","source":"dtypes = {'id':'uint32', 'item_nbr':'int32', 'store_nbr':'int8', 'onpromotion':str}\ndata = {\n    'tra': pd.read_csv('../working/train.csv', dtype=dtypes, parse_dates=['date']),\n    'tes': pd.read_csv('../working/test.csv', dtype=dtypes, parse_dates=['date']),\n    'ite': pd.read_csv('../working/items.csv'),\n    'sto': pd.read_csv('../working/stores.csv'),\n    'trn': pd.read_csv('../working/transactions.csv', parse_dates=['date']),\n    'hol': pd.read_csv('../working/holidays_events.csv', dtype={'transferred':str}, parse_dates=['date']),\n    'oil': pd.read_csv('../working/oil.csv', parse_dates=['date']),\n}\n\nfor k, v in data.items():\n    start_mem = v.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe {} is {:.2f} MB'.format(k, start_mem))\n    NAs = pd.concat([v.isnull().sum()], axis=1, keys=[k])\n    display(NAs[NAs.sum(axis=1) > 0])","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:18:55.695848Z","iopub.execute_input":"2023-12-04T04:18:55.696353Z","iopub.status.idle":"2023-12-04T04:21:03.142345Z","shell.execute_reply.started":"2023-12-04T04:18:55.696318Z","shell.execute_reply":"2023-12-04T04:21:03.140798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# replace N/A with mean value for 'dcoilwtico'\ndata['oil']['dcoilwtico'].fillna(data['oil']['dcoilwtico'].mean(), inplace=True)\n\ndata['tra']['onpromotion'].fillna('False', inplace=True)\ndata['tes']['onpromotion'].fillna('False', inplace=True)\n\nNAs = pd.concat([data['tra'].isnull().sum()], axis=1, keys=['tra'])\ndisplay(NAs[NAs.sum(axis=1) > 0])\n\ndata['tra']['onpromotion'] = data['tra']['onpromotion'].replace({'True': True, 'False': False})\ndata['tes']['onpromotion'] = data['tra']['onpromotion'].replace({'True': True, 'False': False})\n\n# confirm N/As are gone\nNAs = pd.concat([data['oil'].isnull().sum()], axis=1, keys=['oil'])\ndisplay(NAs[NAs.sum(axis=1) > 0])\nNAs = pd.concat([data['tra'].isnull().sum()], axis=1, keys=['tra'])\ndisplay(data['tra'].groupby('onpromotion').count())\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:21:32.273653Z","iopub.execute_input":"2023-12-04T04:21:32.274723Z","iopub.status.idle":"2023-12-04T04:22:22.177447Z","shell.execute_reply.started":"2023-12-04T04:21:32.274649Z","shell.execute_reply":"2023-12-04T04:22:22.175722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib\nimport matplotlib.pyplot as plt\n\ntrain = data['tra']\n#train = pd.read_csv('/kaggle/input/favorita-train-500k/train_500k.csv', dtype=dtypes, parse_dates=['date'])\n\nplt.figure(figsize=(15,6))\n\ntop_unit_sales = train.groupby(['item_nbr']).sum(['unit_sales'])['unit_sales'].sort_values(ascending=False).head(20)\n# display(top_unit_sales)\nax = top_unit_sales.plot.bar(x='item_nbr', y='unit_sales', rot=0)\nplt.ylabel(\"Total Unit Sold\")\nplt.title(\"Top 20 Selling Products\")\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:48:30.575242Z","iopub.execute_input":"2023-12-04T04:48:30.575778Z","iopub.status.idle":"2023-12-04T04:48:40.563511Z","shell.execute_reply.started":"2023-12-04T04:48:30.575738Z","shell.execute_reply":"2023-12-04T04:48:40.560942Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# daily timeseries of top sellers\n\nplt.figure(figsize=(16,8))\n\nfor item_nbr in top_unit_sales.index:\n    x = train[data['tra'].item_nbr == item_nbr]\n    x.groupby(['date']).sum()['unit_sales'].rolling(window=7).mean().plot(x='date',y='unit_sales',rot=0, label=item_nbr)\n    plt.legend(loc=\"upper right\")\nplt.ylabel(\"Total Unit Sold\")\nplt.title(\"Top 10 Selling Products\")\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:48:53.705252Z","iopub.execute_input":"2023-12-04T04:48:53.705715Z","iopub.status.idle":"2023-12-04T04:48:58.836582Z","shell.execute_reply.started":"2023-12-04T04:48:53.705677Z","shell.execute_reply":"2023-12-04T04:48:58.835584Z"},"collapsed":true,"jupyter":{"outputs_hidden":true,"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.tsa.stattools import adfuller\n\nfor item_nbr in top_unit_sales.index:\n    single_item_ts = train[train.item_nbr == item_nbr].groupby('date').sum()['unit_sales']\n    X = single_item_ts.values\n    result = adfuller(X)\n    if result[0] < result[4][\"5%\"]:\n        print (f\"Reject Ho - Time Series is Stationary for item {item_nbr}. ADF: {round(result[0], 2)}, 5%: {round(result[4]['5%'], 2)}.\")\n    else:\n        print (f\"Failed to Reject Ho - Time Series is Non-Stationary for item {item_nbr}. ADF: {round(result[0], 2)}, 5%: {round(result[4]['5%'], 2)}.\")","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:49:29.373632Z","iopub.execute_input":"2023-12-04T04:49:29.374037Z","iopub.status.idle":"2023-12-04T04:49:35.455229Z","shell.execute_reply.started":"2023-12-04T04:49:29.374004Z","shell.execute_reply":"2023-12-04T04:49:35.449001Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_store_context(train_df: pd.DataFrame) -> pd.DataFrame:\n\n    train_df = train_df.merge(data['sto'], on='store_nbr', how='outer')\n    \n    for_dummy = pd.DataFrame\n\n    for col in ['city','state','type','cluster']:\n        for_dummy = train_df.pop(col)\n        train_df = pd.concat([train_df, pd.get_dummies(for_dummy, prefix=col)], axis=1)\n    \n    # knock out the NA rows after the dummies have been concatenated\n    train_df = train_df.loc[train_df['id'].notna()]\n    \n    return train_df","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:22:27.065019Z","iopub.execute_input":"2023-12-04T04:22:27.065590Z","iopub.status.idle":"2023-12-04T04:22:27.077277Z","shell.execute_reply.started":"2023-12-04T04:22:27.065542Z","shell.execute_reply":"2023-12-04T04:22:27.075280Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Union, List\n\ndef regress(reg: PassiveAggressiveRegressor, X: pd.DataFrame, train: bool = False) -> Union[List[float], None]:\n    # if 'train', iterate over items and perform partial fit. Return None\n    # otherwise, perform prediction & return result.\n\n    X['date'] = X['date'].astype(int)\n    \n    if train:\n        items = X['item_nbr'].unique()\n        for i in items:\n            X_item = add_store_context(X.loc[X['item_nbr'] == i])\n            if i % 100 == 0: print('.', end='')\n                \n            y_item = X_item.pop('unit_sales')\n            reg.partial_fit(X_item, y_item)\n    else:\n        y = X.pop('unit_sales')\n        return reg.score(add_store_context(X), y)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:52:43.564692Z","iopub.execute_input":"2023-12-04T04:52:43.565305Z","iopub.status.idle":"2023-12-04T04:52:43.578666Z","shell.execute_reply.started":"2023-12-04T04:52:43.565242Z","shell.execute_reply":"2023-12-04T04:52:43.576882Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data['tra']\nreg = PassiveAggressiveRegressor()\n\n# X_train = train.loc[train['date'] < '2017-07-01']\n# X_val = train.loc[train['date'] >= '2017-07-01']\nX_train = train\n\nregress(reg, X_train, True)\n# print(regress(reg, X_val))\n    \nfilename = f'/kaggle/working/passive-aggressive_{datetime.now().strftime(\"%Y-%m-%d_%H:%M.joblib\")}'\n# filename = f'/kaggle/working/passive-aggressive_partial.joblib'\ndump(reg, filename)\nprint('\\n\\ntraining complete ...')\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:27:00.183805Z","iopub.execute_input":"2023-12-04T04:27:00.184307Z","iopub.status.idle":"2023-12-04T04:40:37.393068Z","shell.execute_reply.started":"2023-12-04T04:27:00.184270Z","shell.execute_reply":"2023-12-04T04:40:37.390377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# filename = '/kaggle/working/passive-aggressive_2023-12-04_02:01.joblib'\nreg: PassiveAggressiveRegressor = load(filename)\ntest = data['tes']\ntest['date'] = test['date'].astype(int)\ntest = add_store_context(test)\npreds = reg.predict(test)\n\nsubmission = pd.DataFrame({'id':test['id'],'unit_sales':preds})\ndisplay(submission)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T05:06:50.955380Z","iopub.execute_input":"2023-12-04T05:06:50.955901Z","iopub.status.idle":"2023-12-04T05:07:07.007538Z","shell.execute_reply.started":"2023-12-04T05:06:50.955855Z","shell.execute_reply":"2023-12-04T05:07:07.006009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reg = load('/kaggle/working/passive-aggressive_partial.joblib')\n# display(regress(reg, data['tra'].loc[data['tra']['date'] >= '2017-07-01']))\n# current score: -0.08634338021353338","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:46:57.869859Z","iopub.execute_input":"2023-12-04T04:46:57.870253Z","iopub.status.idle":"2023-12-04T04:47:07.709682Z","shell.execute_reply.started":"2023-12-04T04:46:57.870222Z","shell.execute_reply":"2023-12-04T04:47:07.708387Z"},"trusted":true},"execution_count":null,"outputs":[]}]}