{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.16","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:39:50.901933Z","iopub.execute_input":"2025-02-01T17:39:50.902224Z","iopub.status.idle":"2025-02-01T17:39:53.507201Z","shell.execute_reply.started":"2025-02-01T17:39:50.902197Z","shell.execute_reply":"2025-02-01T17:39:53.506112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        paths.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:39:53.507910Z","iopub.execute_input":"2025-02-01T17:39:53.508246Z","iopub.status.idle":"2025-02-01T17:39:53.513947Z","shell.execute_reply.started":"2025-02-01T17:39:53.508221Z","shell.execute_reply":"2025-02-01T17:39:53.513112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:39:53.514945Z","iopub.execute_input":"2025-02-01T17:39:53.515194Z","iopub.status.idle":"2025-02-01T17:40:01.343975Z","shell.execute_reply.started":"2025-02-01T17:39:53.515158Z","shell.execute_reply":"2025-02-01T17:40:01.342602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfor filenames in paths:\n    with py7zr.SevenZipFile(filenames, mode='r') as z_ref:\n        z_ref.extractall(path='/kaggle/working')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:40:01.344811Z","iopub.execute_input":"2025-02-01T17:40:01.345079Z","iopub.status.idle":"2025-02-01T17:41:11.145351Z","shell.execute_reply.started":"2025-02-01T17:40:01.345052Z","shell.execute_reply":"2025-02-01T17:41:11.144206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_df.csv`** – the test dataset used to evaluate the model's performance.","metadata":{}},{"cell_type":"code","source":"%%time\ndtype_dict={\"id\":np.uint32,\n            \"store_nbr\":np.uint8,\n            \"item_nbr\":np.uint32,\n            \"unit_sales\":np.float32\n           }\ndata = pd.read_csv(\"/kaggle/working/train.csv\", dtype=dtype_dict)\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:41:11.146264Z","iopub.execute_input":"2025-02-01T17:41:11.146494Z","iopub.status.idle":"2025-02-01T17:42:50.294772Z","shell.execute_reply.started":"2025-02-01T17:41:11.146469Z","shell.execute_reply":"2025-02-01T17:42:50.293473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:42:50.295703Z","iopub.execute_input":"2025-02-01T17:42:50.295965Z","iopub.status.idle":"2025-02-01T17:42:56.976657Z","shell.execute_reply.started":"2025-02-01T17:42:50.295941Z","shell.execute_reply":"2025-02-01T17:42:56.975218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[data[\"date\"] < THRESHOLD_TRAIN_DATE].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:42:56.977357Z","iopub.execute_input":"2025-02-01T17:42:56.977631Z","iopub.status.idle":"2025-02-01T17:50:17.126083Z","shell.execute_reply.started":"2025-02-01T17:42:56.977604Z","shell.execute_reply":"2025-02-01T17:50:17.124863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndtype_dict={\"id\":np.uint32,\n            \"store_nbr\":np.uint8,\n            \"item_nbr\":np.uint32,\n            \"unit_sales\":np.float32\n           }\n\ndf = pd.read_csv(\"/kaggle/working/train_data.csv\", dtype=dtype_dict)\ntest_df = pd.read_csv(\"/kaggle/working/test_data.csv\", dtype=dtype_dict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:50:17.126708Z","iopub.execute_input":"2025-02-01T17:50:17.126949Z","iopub.status.idle":"2025-02-01T17:52:00.193144Z","shell.execute_reply.started":"2025-02-01T17:50:17.126925Z","shell.execute_reply":"2025-02-01T17:52:00.191389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"date\"] = pd.to_datetime(df[\"date\"])\ntest_df[\"date\"] = pd.to_datetime(test_df[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:00.194128Z","iopub.execute_input":"2025-02-01T17:52:00.194406Z","iopub.status.idle":"2025-02-01T17:52:11.617434Z","shell.execute_reply.started":"2025-02-01T17:52:00.194378Z","shell.execute_reply":"2025-02-01T17:52:11.616494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:11.618118Z","iopub.execute_input":"2025-02-01T17:52:11.618350Z","iopub.status.idle":"2025-02-01T17:52:16.517816Z","shell.execute_reply.started":"2025-02-01T17:52:11.618326Z","shell.execute_reply":"2025-02-01T17:52:16.516542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Assuming onpromotion na = -1\ndf.fillna(-1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:16.518833Z","iopub.execute_input":"2025-02-01T17:52:16.519089Z","iopub.status.idle":"2025-02-01T17:52:28.484699Z","shell.execute_reply.started":"2025-02-01T17:52:16.519064Z","shell.execute_reply":"2025-02-01T17:52:28.483356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:28.485727Z","iopub.execute_input":"2025-02-01T17:52:28.486011Z","iopub.status.idle":"2025-02-01T17:52:34.484033Z","shell.execute_reply.started":"2025-02-01T17:52:28.485985Z","shell.execute_reply":"2025-02-01T17:52:34.482647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:34.484804Z","iopub.execute_input":"2025-02-01T17:52:34.485268Z","iopub.status.idle":"2025-02-01T17:52:35.021152Z","shell.execute_reply.started":"2025-02-01T17:52:34.485237Z","shell.execute_reply":"2025-02-01T17:52:35.019913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:35.022350Z","iopub.execute_input":"2025-02-01T17:52:35.022604Z","iopub.status.idle":"2025-02-01T17:52:35.044823Z","shell.execute_reply.started":"2025-02-01T17:52:35.022581Z","shell.execute_reply":"2025-02-01T17:52:35.043643Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"execution":{"iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z"}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\nholiday_events['date'] = pd.to_datetime(holiday_events['date'])\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:35.045639Z","iopub.execute_input":"2025-02-01T17:52:35.045878Z","iopub.status.idle":"2025-02-01T17:52:35.059864Z","shell.execute_reply.started":"2025-02-01T17:52:35.045855Z","shell.execute_reply":"2025-02-01T17:52:35.058663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(holiday_events[['date', 'type', 'locale']], on='date', how='left')\ntest_df = test_df.merge(holiday_events[['date', 'type', 'locale']], on='date', how='left')\n\ndf['is_holiday'] = df['type'].notnull().astype(int)\ntest_df['is_holiday'] = test_df['type'].notnull().astype(int)\n\ndf['type'] = df['type'].fillna('None')\ntest_df['type'] = test_df['type'].fillna('None')\ndf['locale'] = df['locale'].fillna('None')\ntest_df['locale'] = test_df['locale'].fillna('None')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:52:35.060850Z","iopub.execute_input":"2025-02-01T17:52:35.061093Z","iopub.status.idle":"2025-02-01T17:53:16.730084Z","shell.execute_reply.started":"2025-02-01T17:52:35.061070Z","shell.execute_reply":"2025-02-01T17:53:16.728774Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Holiday types and locales","metadata":{}},{"cell_type":"code","source":"holiday_sales = df.groupby('type')['unit_sales'].sum()\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x=holiday_sales.index, y=holiday_sales.values, palette='viridis')\nplt.title('Unit Sales by Holiday Type')\nplt.xlabel('Holiday Type')\nplt.ylabel('Total Unit Sales')\nplt.show()\n\nlocale_sales = df.groupby('locale')['unit_sales'].sum()\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x=locale_sales.index, y=locale_sales.values, palette='viridis')\nplt.title('Unit Sales by Locale')\nplt.xlabel('Locale')\nplt.ylabel('Total Unit Sales')\nplt.show()\n\n%xdel holiday_sales\n%xdel locale_sales\n\nholiday_sales = df.groupby('type')['unit_sales'].mean()\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x=holiday_sales.index, y=holiday_sales.values, palette='viridis')\nplt.title('Unit Sales by Holiday Type')\nplt.xlabel('Holiday Type')\nplt.ylabel('Mean Unit Sales')\nplt.show()\n\nlocale_sales = df.groupby('locale')['unit_sales'].mean()\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x=locale_sales.index, y=locale_sales.values, palette='viridis')\nplt.title('Unit Sales by Locale')\nplt.xlabel('Locale')\nplt.ylabel('Mean Unit Sales')\nplt.show()\n\n%xdel holiday_sales\n%xdel locale_sales","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:53:16.730745Z","iopub.execute_input":"2025-02-01T17:53:16.730991Z","iopub.status.idle":"2025-02-01T17:53:52.076656Z","shell.execute_reply.started":"2025-02-01T17:53:16.730967Z","shell.execute_reply":"2025-02-01T17:53:52.075343Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**My opinion - whilst there is indeed difference in sales, it's pretty much unnoticeable. Besides, there is a huge imbalance between total numbers, so at least in this category I would say it's not that important.**","metadata":{}},{"cell_type":"markdown","source":"### Holiday vs Normal days","metadata":{}},{"cell_type":"code","source":"avg_sales_holiday = df[df['is_holiday'] == 1]['unit_sales'].mean()\navg_sales_non_holiday = df[df['is_holiday'] == 0]['unit_sales'].mean()\n\nplt.figure(figsize=(8, 6))\nsns.barplot(x=['Holiday', 'Non-Holiday'], y=[avg_sales_holiday, avg_sales_non_holiday], palette='viridis')\nplt.title('Average Unit Sales: Holiday vs. Non-Holiday')\nplt.xlabel('Day Type')\nplt.ylabel('Average Unit Sales')\nplt.show()\n\n%xdel avg_sales_holiday\n%xdel avg_sales_non_holiday","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:53:52.077442Z","iopub.execute_input":"2025-02-01T17:53:52.077659Z","iopub.status.idle":"2025-02-01T17:54:04.641168Z","shell.execute_reply.started":"2025-02-01T17:53:52.077637Z","shell.execute_reply":"2025-02-01T17:54:04.639543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Seems pretty meh for me - we are talking about using imbalanced feature to decide between 8 or 9, which is not that much important IMO. It might complicate things too much.","metadata":{}},{"cell_type":"code","source":"# # too lazy for using sklearn dummy/one-hot encoding - UPD: forgot that catboost does brrrr\n# types = ['National', 'Regional', 'Local', 'None']\n# for t in types:\n#     train_data[f'is_{t.lower()}_holiday'] = (train_data['type'] == t).astype(int)\n#     test_data[f'is_{t.lower()}_holiday'] = (test_data['type'] == t).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:54:04.642042Z","iopub.execute_input":"2025-02-01T17:54:04.642273Z","iopub.status.idle":"2025-02-01T17:54:04.647341Z","shell.execute_reply.started":"2025-02-01T17:54:04.642251Z","shell.execute_reply":"2025-02-01T17:54:04.645434Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{}},{"cell_type":"markdown","source":"## WARNING - This stuff crashes runtime due to RAM issues, I am sick of it\nTherefore, I can't proceed with this step - sorry. In exchange I can tell that it *might* be solved with chunking and libs that support chunking in pandas-like interfaces(dask, polaris maybe, feather), but it will take a significant amount of trial and error to find a good fit even if I will be able to merge the data - I dunno what will be the final size of the dataset after that after all. *Also it will require to rewrite all the current logic, which is pretty time-consuming - so I consider it as proff by enterprise-like argumentation :)*","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\nprint(items.shape)\nitems.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:54:04.648116Z","iopub.execute_input":"2025-02-01T17:54:04.648345Z","iopub.status.idle":"2025-02-01T17:54:04.667960Z","shell.execute_reply.started":"2025-02-01T17:54:04.648320Z","shell.execute_reply":"2025-02-01T17:54:04.666499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(items, on='item_nbr', how='left')\ntest_df = test_df.merge(items, on='item_nbr', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:54:04.669007Z","iopub.execute_input":"2025-02-01T17:54:04.669251Z","iopub.status.idle":"2025-02-01T17:54:37.303112Z","shell.execute_reply.started":"2025-02-01T17:54:04.669227Z","shell.execute_reply":"2025-02-01T17:54:37.302029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"family_sales = df.groupby('family')['unit_sales'].sum()\n\nplt.figure(figsize=(12, 8))\nsns.barplot(x=family_sales.index, y=family_sales.values, palette='viridis')\nplt.title('Total Unit Sales by Item Family')\nplt.xlabel('Item Family')\nplt.ylabel('Total Unit Sales')\nplt.xticks(rotation=90)\nplt.show()\n\n%xdel family_sales","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:54:37.303934Z","iopub.execute_input":"2025-02-01T17:54:37.304163Z","iopub.status.idle":"2025-02-01T17:54:47.227058Z","shell.execute_reply.started":"2025-02-01T17:54:37.304139Z","shell.execute_reply":"2025-02-01T17:54:47.226014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_sales = df.groupby('class')['unit_sales'].sum()\n\ntop = class_sales.nlargest(15) # Top 15\nplt.figure(figsize=(12, 8))\nsns.barplot(x=top.index, y=top.values, palette='viridis')\nplt.show()\n%xdel class_sales, top","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:04.248605Z","iopub.execute_input":"2025-02-01T17:59:04.248987Z","iopub.status.idle":"2025-02-01T17:59:06.457764Z","shell.execute_reply.started":"2025-02-01T17:59:04.248956Z","shell.execute_reply":"2025-02-01T17:59:06.456283Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Makes a lot of sense to keep both of them.","metadata":{}},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{}},{"cell_type":"markdown","source":"**Note: Oil prices are having pretty small correlation and contain spikes, so I see no reason to try feature engineering it here**","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\nprint(oil.shape)\noil.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:29.730626Z","iopub.execute_input":"2025-02-01T17:59:29.730991Z","iopub.status.idle":"2025-02-01T17:59:29.742693Z","shell.execute_reply.started":"2025-02-01T17:59:29.730962Z","shell.execute_reply":"2025-02-01T17:59:29.741849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:29.916044Z","iopub.execute_input":"2025-02-01T17:59:29.916326Z","iopub.status.idle":"2025-02-01T17:59:29.923158Z","shell.execute_reply.started":"2025-02-01T17:59:29.916301Z","shell.execute_reply":"2025-02-01T17:59:29.921819Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Well, 43 is not so big for NA's, let's see the price plot and compare it with the data one from the beginning(wouldn't copy it here because RAM goes brrr):","metadata":{}},{"cell_type":"code","source":"oil['date'] = pd.to_datetime(oil['date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:30.272972Z","iopub.execute_input":"2025-02-01T17:59:30.273282Z","iopub.status.idle":"2025-02-01T17:59:30.278844Z","shell.execute_reply.started":"2025-02-01T17:59:30.273258Z","shell.execute_reply":"2025-02-01T17:59:30.277641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axis = plt.subplots(figsize=(12, 6))\n\noil[(oil[\"date\"] >= pd.to_datetime(\"2012-12-01\"))&(oil[\"date\"] < pd.to_datetime(\"2017-08-01\"))].fillna(0).plot(x='date', y='dcoilwtico', kind='bar', color='skyblue', ax=axis)\naxis.set_title('Oil Prices Data for Each day', fontsize=14)\naxis.set_xlabel('Day', fontsize=12)\naxis.set_ylabel('Price', fontsize=12)\n# axis.set_xticklabels(axis.get_xticklabels(), rotation=45)\naxis.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:30.462468Z","iopub.execute_input":"2025-02-01T17:59:30.462718Z","iopub.status.idle":"2025-02-01T17:59:37.924290Z","shell.execute_reply.started":"2025-02-01T17:59:30.462695Z","shell.execute_reply":"2025-02-01T17:59:37.922990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Allegedly, even without looking into each item we would see about the same sales distribution as in oil prices(if there is correlation) - scale doesn't change it, if all items have the same correlation with oil prices, we would see that as the sum of these trends/sales.","metadata":{}},{"cell_type":"code","source":"fig, axis = plt.subplots(figsize=(12, 6))\n\ndf.groupby('date')['unit_sales'].mean().plot(x='date', y='unit_sales', kind='bar', color='skyblue', ax=axis)\naxis.set_title('Unit sales Data for Each day (no separation by items)', fontsize=14)\naxis.set_xlabel('Day', fontsize=12)\naxis.set_ylabel('Price', fontsize=12)\n# axis.set_xticklabels(axis.get_xticklabels(), rotation=45)\naxis.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:37.925216Z","iopub.execute_input":"2025-02-01T17:59:37.925447Z","iopub.status.idle":"2025-02-01T17:59:50.284444Z","shell.execute_reply.started":"2025-02-01T17:59:37.925424Z","shell.execute_reply":"2025-02-01T17:59:50.283198Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Even this rough look is enough to conclude that there is no correlation between data, furthermore, oil prices would provide spiking in some regions that don't have the same sesasonality. Thus, I decide to omit it.","metadata":{}},{"cell_type":"code","source":"%xdel oil","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:50.285364Z","iopub.execute_input":"2025-02-01T17:59:50.285704Z","iopub.status.idle":"2025-02-01T17:59:50.290013Z","shell.execute_reply.started":"2025-02-01T17:59:50.285677Z","shell.execute_reply":"2025-02-01T17:59:50.288936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.\n","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:50.290917Z","iopub.execute_input":"2025-02-01T17:59:50.291205Z","iopub.status.idle":"2025-02-01T17:59:50.309054Z","shell.execute_reply.started":"2025-02-01T17:59:50.291168Z","shell.execute_reply":"2025-02-01T17:59:50.308082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(stores, on='store_nbr', how='left')\ntest_df = test_df.merge(stores, on='store_nbr', how='left')\n# Another crash","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T17:59:50.309706Z","iopub.execute_input":"2025-02-01T17:59:50.309943Z","iopub.status.idle":"2025-02-01T18:00:31.582791Z","shell.execute_reply.started":"2025-02-01T17:59:50.309919Z","shell.execute_reply":"2025-02-01T18:00:31.581536Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"markdown","source":"## Baseline model","metadata":{}},{"cell_type":"markdown","source":"As a baseline, I want to use catboost, since it supports many features out of the box, which should be good. Also let's consider a small GridSearch for Hyperparameters, since we are not doing much Feature Engineering and processing here ","metadata":{}},{"cell_type":"markdown","source":"Firstly, define the NWRMSLE function, I haven't found one in any of the libs provided(mb skill issue with finding it in the documentation), so here's one made by me:","metadata":{}},{"cell_type":"code","source":"def nwrmsle(y_true, y_pred, W=None):\n    y_true = y_true.clip(0, y_true.max()) # To prevent log <= 0\n    y_pred = y_pred.clip(0, y_pred.max()) \n    \n    if W is None:\n        W = np.ones_like(y_true)\n    \n    weighted_msle = np.nansum(W * ((np.log1p(y_pred) - np.log1p(y_true)) ** 2)) / np.sum(W)\n    \n    return np.sqrt(weighted_msle)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:00:31.583751Z","iopub.execute_input":"2025-02-01T18:00:31.584002Z","iopub.status.idle":"2025-02-01T18:00:31.589398Z","shell.execute_reply.started":"2025-02-01T18:00:31.583977Z","shell.execute_reply":"2025-02-01T18:00:31.588324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:00:31.590142Z","iopub.execute_input":"2025-02-01T18:00:31.590380Z","iopub.status.idle":"2025-02-01T18:01:03.506282Z","shell.execute_reply.started":"2025-02-01T18:00:31.590352Z","shell.execute_reply":"2025-02-01T18:01:03.505129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:01:03.507210Z","iopub.execute_input":"2025-02-01T18:01:03.507443Z","iopub.status.idle":"2025-02-01T18:01:04.265510Z","shell.execute_reply.started":"2025-02-01T18:01:03.507420Z","shell.execute_reply":"2025-02-01T18:01:04.264271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['year'] = df['date'].dt.year.astype(np.uint16)\ndf['month'] = df['date'].dt.month.astype(np.uint8)\ndf['day'] = df['date'].dt.day.astype(np.uint8)\ndf['dayofweek'] = df['date'].dt.dayofweek","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:01:04.266486Z","iopub.execute_input":"2025-02-01T18:01:04.266715Z","iopub.status.idle":"2025-02-01T18:01:16.286269Z","shell.execute_reply.started":"2025-02-01T18:01:04.266692Z","shell.execute_reply":"2025-02-01T18:01:16.285027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit\ntscv = TimeSeriesSplit(n_splits=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:02:46.018331Z","iopub.execute_input":"2025-02-01T18:02:46.018659Z","iopub.status.idle":"2025-02-01T18:02:46.315344Z","shell.execute_reply.started":"2025-02-01T18:02:46.018634Z","shell.execute_reply":"2025-02-01T18:02:46.314473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install catboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:03:08.604262Z","iopub.execute_input":"2025-02-01T18:03:08.604634Z","iopub.status.idle":"2025-02-01T18:03:31.322688Z","shell.execute_reply.started":"2025-02-01T18:03:08.604609Z","shell.execute_reply":"2025-02-01T18:03:31.320891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\nfrom catboost import CatBoostRegressor, Pool\n\n# # Just in case\n# # df.fillna(0, inplace=True)\n\n# # df['date'] = pd.to_datetime(df['date'])\n# # df['year'] = df['date'].dt.year\n# # df['month'] = df['date'].dt.month\n# # df['day'] = df['date'].dt.day\n# # df['dayofweek'] = df['date'].dt.dayofweek\n# # df['weekofyear'] = df['date'].dt.isocalendar().week\n\n# # features = ['store_nbr', 'item_nbr', 'year', 'month', 'day', 'dayofweek', 'weekofyear',] #  'family', 'class', 'perishable', 'cluster'\n# # target = 'unit_sales'\n\n# # cat_features = ['family', 'class', 'perishable', 'cluster']\n# # cat_features = []\n\n# # Perishable columns have weights, which have been described in dataset description\ndf['weight'] = df['perishable'].apply(lambda x: 1.25 if x == 1 else 1.0)\n# features = ['store_nbr', 'item_nbr', 'year', 'month', 'day', 'dayofweek', 'onpromotion',] #  'family', 'class', 'perishable', 'cluster'\n# cat_features = ['onpromotion']\n# target = 'unit_sales'\n# _, X_val, _, y_val = train_test_split(df[features], df[target], test_size=0.2, random_state=42)\n\n# # train_pool = Pool(data=X_train, label=y_train, cat_features=cat_features)\n# # val_pool = Pool(data=X_val, label=y_val, cat_features=cat_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:03:31.323911Z","iopub.execute_input":"2025-02-01T18:03:31.324265Z","iopub.status.idle":"2025-02-01T18:03:58.534743Z","shell.execute_reply.started":"2025-02-01T18:03:31.324225Z","shell.execute_reply":"2025-02-01T18:03:58.533497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer\n\nnwrmsle_scorer = make_scorer(nwrmsle, greater_is_better=False, W=None) # no perishables (. UPD: Lazy to add weights (\n\nparams = {\n    'depth': [4, 6, 8],\n    'learning_rate': [0.01, 0.1, 0.2],\n    'iterations': [500, 1000], # early stopping goes brr, so I assume no need to make it too complex\n    'l2_leaf_reg': [1, 3, 5]\n}\n\nmodel = CatBoostRegressor(loss_function='RMSE', verbose=100, random_state=42)\n\ngrid_search = GridSearchCV(estimator=model, param_grid=params, scoring=nwrmsle_scorer, cv=tscv, n_jobs=-1, verbose=2) # 'neg_mean_squared_log_error'\nfeatures = ['store_nbr', 'item_nbr', 'year', 'month', 'day', 'dayofweek', 'onpromotion','family', 'class', 'perishable', 'cluster', 'weight'] #  'family', 'class', 'perishable', 'cluster'\ncat_features = ['onpromotion', 'family', 'class', 'perishable']\ntarget = 'unit_sales'\n\n# batch_size = 300000\n# for i in range(0, len(df), batch_size):\n#     batch = df[i:i + batch_size]\ngrid_search.fit(df[features], df[target], cat_features=cat_features)\n\nprint(\"Best parameters:\", grid_search.best_params_)\nprint(\"Best NWRMSLE score:\", -grid_search.best_score_)\n\n# best_model = grid_search.best_estimator_\n# best_model.fit(train_data[features], train_data[target], cat_features=cat_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:14:59.253068Z","iopub.execute_input":"2025-02-01T18:14:59.253422Z","iopub.status.idle":"2025-02-01T18:15:40.993765Z","shell.execute_reply.started":"2025-02-01T18:14:59.253394Z","shell.execute_reply":"2025-02-01T18:15:40.992088Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"No validation this time - RAM stuff, also I think CV is pretty much sufficient here","metadata":{}},{"cell_type":"code","source":"# test_df['date'] = pd.to_datetime(test_df['date'])\ntest_df['year'] = test_df['date'].dt.year\ntest_df['month'] = test_df['date'].dt.month\ntest_df['day'] = test_df['date'].dt.day\ntest_df['dayofweek'] = test_df['date'].dt.dayofweek\n\ntest_df['weight'] = test_df['perishable'].apply(lambda x: 1.25 if x == 1 else 1.0)\n\n# Merge with stores and items\n# test_df = test_df.merge(stores, on='store_nbr', how='left')\n# test_df = test_df.merge(items, on='item_nbr', how='left')\n\n# Handle missing values\ntest_df.fillna(0, inplace=True)\n\ntest_predictions = grid_search.best_estimator_.predict(test_df[features])\n\n# Save predictions\nsubmission = pd.DataFrame({'id': test_df['id'], 'unit_sales': test_predictions})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-01T18:15:48.123281Z","iopub.execute_input":"2025-02-01T18:15:48.123643Z","iopub.status.idle":"2025-02-01T18:15:49.871699Z","shell.execute_reply.started":"2025-02-01T18:15:48.123609Z","shell.execute_reply":"2025-02-01T18:15:49.870347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}