{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"},{"sourceId":10369576,"sourceType":"datasetVersion","datasetId":6422933}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:17:03.693804Z","iopub.execute_input":"2025-01-07T12:17:03.694170Z","iopub.status.idle":"2025-01-07T12:17:05.570214Z","shell.execute_reply.started":"2025-01-07T12:17:03.694143Z","shell.execute_reply":"2025-01-07T12:17:05.569349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/input/favorita-grocery-sales-forecasting'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        paths.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:17:05.571268Z","iopub.execute_input":"2025-01-07T12:17:05.571687Z","iopub.status.idle":"2025-01-07T12:17:05.580329Z","shell.execute_reply.started":"2025-01-07T12:17:05.571651Z","shell.execute_reply":"2025-01-07T12:17:05.579436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T11:29:35.639750Z","iopub.execute_input":"2025-01-07T11:29:35.640104Z","iopub.status.idle":"2025-01-07T11:29:44.315370Z","shell.execute_reply.started":"2025-01-07T11:29:35.640081Z","shell.execute_reply":"2025-01-07T11:29:44.314395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfor filenames in paths:\n    with py7zr.SevenZipFile(filenames, mode='r') as z_ref:\n        z_ref.extractall(path='/kaggle/working')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T11:29:44.316884Z","iopub.execute_input":"2025-01-07T11:29:44.317225Z","iopub.status.idle":"2025-01-07T11:31:11.950223Z","shell.execute_reply.started":"2025-01-07T11:29:44.317186Z","shell.execute_reply":"2025-01-07T11:31:11.949315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_data.csv`** – the test dataset used to evaluate the model's performance.","metadata":{}},{"cell_type":"code","source":"%%time\ndata = pd.read_csv(\"/kaggle/working/train.csv\")\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T09:45:31.662960Z","iopub.execute_input":"2025-01-04T09:45:31.663453Z","iopub.status.idle":"2025-01-04T09:47:11.361248Z","shell.execute_reply.started":"2025-01-04T09:45:31.663365Z","shell.execute_reply":"2025-01-04T09:47:11.359785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T09:47:11.363206Z","iopub.execute_input":"2025-01-04T09:47:11.363623Z","iopub.status.idle":"2025-01-04T09:47:19.037897Z","shell.execute_reply.started":"2025-01-04T09:47:11.363580Z","shell.execute_reply":"2025-01-04T09:47:19.036705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[data[\"date\"] < THRESHOLD_TRAIN_DATE].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T09:47:19.039173Z","iopub.execute_input":"2025-01-04T09:47:19.039498Z","iopub.status.idle":"2025-01-04T09:54:43.480986Z","shell.execute_reply.started":"2025-01-04T09:47:19.039471Z","shell.execute_reply":"2025-01-04T09:54:43.478949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"usecols = [\"date\", \"store_nbr\", \"item_nbr\", \"unit_sales\", \"onpromotion\"]\ndtypes = {\n    \"date\": 'str',\n    \"store_nbr\": 'int',\n    \"item_nbr\": 'int',\n    \"unit_sales\": 'float',\n    \"onpromotion\": 'O'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:17:09.132520Z","iopub.execute_input":"2025-01-07T12:17:09.132902Z","iopub.status.idle":"2025-01-07T12:17:09.137395Z","shell.execute_reply.started":"2025-01-07T12:17:09.132876Z","shell.execute_reply":"2025-01-07T12:17:09.136195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\n    \"/kaggle/input/zipped-favorita-grocery-sales/train_data/train_data.csv\", \n    usecols=usecols, \n    dtype=dtypes, \n    parse_dates=['date']\n)\ntest_df = pd.read_csv(\n    \"/kaggle/input/zipped-favorita-grocery-sales/test_data/test_data.csv\", \n    usecols=usecols, \n    dtype=dtypes, \n    parse_dates=['date']\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:17:09.340257Z","iopub.execute_input":"2025-01-07T12:17:09.340738Z","iopub.status.idle":"2025-01-07T12:19:32.358631Z","shell.execute_reply.started":"2025-01-07T12:17:09.340698Z","shell.execute_reply":"2025-01-07T12:19:32.357412Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T11:06:19.976313Z","iopub.execute_input":"2025-01-04T11:06:19.976654Z","iopub.status.idle":"2025-01-04T11:06:26.205823Z","shell.execute_reply.started":"2025-01-04T11:06:19.976618Z","shell.execute_reply":"2025-01-04T11:06:26.204769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T11:06:26.206934Z","iopub.execute_input":"2025-01-04T11:06:26.207223Z","iopub.status.idle":"2025-01-04T11:06:27.380368Z","shell.execute_reply.started":"2025-01-04T11:06:26.207200Z","shell.execute_reply":"2025-01-04T11:06:27.379346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T11:06:27.382947Z","iopub.execute_input":"2025-01-04T11:06:27.383227Z","iopub.status.idle":"2025-01-04T11:06:27.400085Z","shell.execute_reply.started":"2025-01-04T11:06:27.383204Z","shell.execute_reply":"2025-01-04T11:06:27.399007Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We will use only last year of data because of memory concerns","metadata":{}},{"cell_type":"code","source":"THRESHOLD_TRAIN_DATE_LOWER = pd.to_datetime(\"2017-01-01\")\ndf = df[df[\"date\"] >= THRESHOLD_TRAIN_DATE_LOWER]\ndf = df.reset_index(drop=True)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:32.360250Z","iopub.execute_input":"2025-01-07T12:19:32.360708Z","iopub.status.idle":"2025-01-07T12:19:34.497789Z","shell.execute_reply.started":"2025-01-07T12:19:32.360663Z","shell.execute_reply":"2025-01-07T12:19:34.496917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"execution":{"iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z"}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:34.499543Z","iopub.execute_input":"2025-01-07T12:19:34.499807Z","iopub.status.idle":"2025-01-07T12:19:34.517966Z","shell.execute_reply.started":"2025-01-07T12:19:34.499785Z","shell.execute_reply":"2025-01-07T12:19:34.516967Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"According to data description transferred holidays are more like normal days, so we should probably skip those\n\nAlso rows with type \"Work Day\" are actually work days that are meant to payback \"Bridge\" holidays, so we should also skip those rows","metadata":{}},{"cell_type":"code","source":"holiday_events = holiday_events[holiday_events.transferred == False]\nholiday_events = holiday_events[holiday_events.type != \"Work Day\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:34.519491Z","iopub.execute_input":"2025-01-07T12:19:34.519765Z","iopub.status.idle":"2025-01-07T12:19:34.525749Z","shell.execute_reply.started":"2025-01-07T12:19:34.519743Z","shell.execute_reply":"2025-01-07T12:19:34.524703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_events['date'] = pd.to_datetime(holiday_events['date'])\nholiday_events = holiday_events.rename(columns={\"type\": \"holiday_type\"})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:34.526721Z","iopub.execute_input":"2025-01-07T12:19:34.527007Z","iopub.status.idle":"2025-01-07T12:19:34.543436Z","shell.execute_reply.started":"2025-01-07T12:19:34.526984Z","shell.execute_reply":"2025-01-07T12:19:34.542342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(holiday_events[['date', 'holiday_type', 'locale']], on='date', how='left')\ntest_df = test_df.merge(holiday_events[['date', 'holiday_type', 'locale']], on='date', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:34.544619Z","iopub.execute_input":"2025-01-07T12:19:34.545000Z","iopub.status.idle":"2025-01-07T12:19:38.508220Z","shell.execute_reply.started":"2025-01-07T12:19:34.544963Z","shell.execute_reply":"2025-01-07T12:19:38.506987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['is_holiday'] = test_df['holiday_type'].notna().astype(int)\ndf['is_holiday'] = df['holiday_type'].notna().astype(int)\n\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:38.509539Z","iopub.execute_input":"2025-01-07T12:19:38.509939Z","iopub.status.idle":"2025-01-07T12:19:39.189515Z","shell.execute_reply.started":"2025-01-07T12:19:38.509900Z","shell.execute_reply":"2025-01-07T12:19:39.188371Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\nprint(items.shape)\nitems.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:39.192051Z","iopub.execute_input":"2025-01-07T12:19:39.192313Z","iopub.status.idle":"2025-01-07T12:19:39.209398Z","shell.execute_reply.started":"2025-01-07T12:19:39.192291Z","shell.execute_reply":"2025-01-07T12:19:39.208334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(items, on='item_nbr', how='left')\ntest_df = test_df.merge(items, on='item_nbr', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:39.211401Z","iopub.execute_input":"2025-01-07T12:19:39.211740Z","iopub.status.idle":"2025-01-07T12:19:46.086783Z","shell.execute_reply.started":"2025-01-07T12:19:39.211715Z","shell.execute_reply":"2025-01-07T12:19:46.085836Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\nprint(oil.shape)\noil.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:46.087728Z","iopub.execute_input":"2025-01-07T12:19:46.087972Z","iopub.status.idle":"2025-01-07T12:19:46.101658Z","shell.execute_reply.started":"2025-01-07T12:19:46.087952Z","shell.execute_reply":"2025-01-07T12:19:46.100551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil['date'] = pd.to_datetime(oil['date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:46.102827Z","iopub.execute_input":"2025-01-07T12:19:46.103195Z","iopub.status.idle":"2025-01-07T12:19:46.113624Z","shell.execute_reply.started":"2025-01-07T12:19:46.103153Z","shell.execute_reply":"2025-01-07T12:19:46.112529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dates = df[['date']]\ntest_df_dates = test_df[['date']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:46.114659Z","iopub.execute_input":"2025-01-07T12:19:46.114964Z","iopub.status.idle":"2025-01-07T12:19:46.197038Z","shell.execute_reply.started":"2025-01-07T12:19:46.114928Z","shell.execute_reply":"2025-01-07T12:19:46.196122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dates = df_dates.merge(oil, on='date', how='left', copy=False)\ntest_df_dates = test_df_dates.merge(oil, on='date', how='left', copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:46.198082Z","iopub.execute_input":"2025-01-07T12:19:46.198430Z","iopub.status.idle":"2025-01-07T12:19:47.986478Z","shell.execute_reply.started":"2025-01-07T12:19:46.198389Z","shell.execute_reply":"2025-01-07T12:19:47.985723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dates = df_dates.sort_values('date')\ntest_df_dates = test_df_dates.sort_values('date')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:47.987340Z","iopub.execute_input":"2025-01-07T12:19:47.987661Z","iopub.status.idle":"2025-01-07T12:19:49.790114Z","shell.execute_reply.started":"2025-01-07T12:19:47.987635Z","shell.execute_reply":"2025-01-07T12:19:49.788909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Fill missing values","metadata":{}},{"cell_type":"code","source":"df_dates.dcoilwtico = df_dates.dcoilwtico.ffill()\ndf_dates.dcoilwtico = df_dates.dcoilwtico.bfill()\n\ntest_df_dates.dcoilwtico = test_df_dates.dcoilwtico.ffill()\ntest_df_dates.dcoilwtico = test_df_dates.dcoilwtico.bfill()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:49.791285Z","iopub.execute_input":"2025-01-07T12:19:49.791736Z","iopub.status.idle":"2025-01-07T12:19:50.196390Z","shell.execute_reply.started":"2025-01-07T12:19:49.791694Z","shell.execute_reply":"2025-01-07T12:19:50.195608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dates = df_dates.sort_index()\ntest_df_dates = test_df_dates.sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:50.197380Z","iopub.execute_input":"2025-01-07T12:19:50.197767Z","iopub.status.idle":"2025-01-07T12:19:52.053688Z","shell.execute_reply.started":"2025-01-07T12:19:50.197728Z","shell.execute_reply":"2025-01-07T12:19:52.052569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['oil_price'] = df_dates.dcoilwtico\ntest_df['oil_price'] = test_df_dates.dcoilwtico","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:52.054787Z","iopub.execute_input":"2025-01-07T12:19:52.055143Z","iopub.status.idle":"2025-01-07T12:19:52.206946Z","shell.execute_reply.started":"2025-01-07T12:19:52.055102Z","shell.execute_reply":"2025-01-07T12:19:52.205586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:52.208115Z","iopub.execute_input":"2025-01-07T12:19:52.208558Z","iopub.status.idle":"2025-01-07T12:19:52.222479Z","shell.execute_reply.started":"2025-01-07T12:19:52.208516Z","shell.execute_reply":"2025-01-07T12:19:52.221381Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.\n","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:52.223506Z","iopub.execute_input":"2025-01-07T12:19:52.223825Z","iopub.status.idle":"2025-01-07T12:19:52.244235Z","shell.execute_reply.started":"2025-01-07T12:19:52.223791Z","shell.execute_reply":"2025-01-07T12:19:52.243259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores = stores.rename(columns={\"type\": \"store_type\"})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:52.245203Z","iopub.execute_input":"2025-01-07T12:19:52.245610Z","iopub.status.idle":"2025-01-07T12:19:52.250171Z","shell.execute_reply.started":"2025-01-07T12:19:52.245575Z","shell.execute_reply":"2025-01-07T12:19:52.249387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(stores, on='store_nbr', how='left')\ntest_df = test_df.merge(stores, on='store_nbr', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:19:52.251065Z","iopub.execute_input":"2025-01-07T12:19:52.251284Z","iopub.status.idle":"2025-01-07T12:20:00.881539Z","shell.execute_reply.started":"2025-01-07T12:19:52.251266Z","shell.execute_reply":"2025-01-07T12:20:00.880765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"markdown","source":"Metric\n$$NWRMSLE = \\sqrt{ \\frac{\\sum_{i=1}^n w_i \\left( \\ln(\\hat{y}_i + 1) - \\ln(y_i +1)  \\right)^2  }{\\sum_{i=1}^n w_i}}$$","metadata":{}},{"cell_type":"markdown","source":"Get weights for test data","metadata":{}},{"cell_type":"code","source":"weights = test_df.perishable.astype(float).values * 0.25 + 1\nweights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:00.882391Z","iopub.execute_input":"2025-01-07T12:20:00.882652Z","iopub.status.idle":"2025-01-07T12:20:00.900277Z","shell.execute_reply.started":"2025-01-07T12:20:00.882631Z","shell.execute_reply":"2025-01-07T12:20:00.899508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_log(arr):\n    x = np.ma.log(arr)\n    return x.filled(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:00.903549Z","iopub.execute_input":"2025-01-07T12:20:00.903792Z","iopub.status.idle":"2025-01-07T12:20:00.907589Z","shell.execute_reply.started":"2025-01-07T12:20:00.903771Z","shell.execute_reply":"2025-01-07T12:20:00.906849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def nwrmsle(y_pred, y_true, weights):\n    return np.sqrt(np.sum(weights * (compute_log(y_pred + 1) - compute_log(y_true + 1)) ** 2) / np.sum(weights))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:00.908953Z","iopub.execute_input":"2025-01-07T12:20:00.909201Z","iopub.status.idle":"2025-01-07T12:20:00.922281Z","shell.execute_reply.started":"2025-01-07T12:20:00.909181Z","shell.execute_reply":"2025-01-07T12:20:00.921552Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**1. Baseline Model**\n\nSince we are faced with regression task we will use dummy regressor as our baseline","metadata":{}},{"cell_type":"code","source":"from sklearn.dummy import DummyRegressor\n\ndummy_regr = DummyRegressor(strategy=\"mean\")\ndummy_regr.fit(df, df['unit_sales'].values)\n\ndummy_prediction = dummy_regr.predict(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:00.923106Z","iopub.execute_input":"2025-01-07T12:20:00.923337Z","iopub.status.idle":"2025-01-07T12:20:01.164421Z","shell.execute_reply.started":"2025-01-07T12:20:00.923318Z","shell.execute_reply":"2025-01-07T12:20:01.163617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dummy_score = nwrmsle(dummy_prediction, test_df['unit_sales'].values, weights)\ndummy_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:01.165357Z","iopub.execute_input":"2025-01-07T12:20:01.165667Z","iopub.status.idle":"2025-01-07T12:20:01.289378Z","shell.execute_reply.started":"2025-01-07T12:20:01.165636Z","shell.execute_reply":"2025-01-07T12:20:01.288498Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**2. Feature engineering**\n\nWe've already added 'is_holiday' column and oil prices","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:01.290327Z","iopub.execute_input":"2025-01-07T12:20:01.290636Z","iopub.status.idle":"2025-01-07T12:20:01.305176Z","shell.execute_reply.started":"2025-01-07T12:20:01.290613Z","shell.execute_reply":"2025-01-07T12:20:01.304176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:01.306064Z","iopub.execute_input":"2025-01-07T12:20:01.306319Z","iopub.status.idle":"2025-01-07T12:20:06.653363Z","shell.execute_reply.started":"2025-01-07T12:20:01.306296Z","shell.execute_reply":"2025-01-07T12:20:06.652182Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's fill nan columns","metadata":{}},{"cell_type":"code","source":"df['holiday_type'] = df.holiday_type.fillna('Not holiday')\ntest_df['holiday_type'] = test_df.holiday_type.fillna('Not holiday')\n\ndf['locale'] = df.locale.fillna('None')\ntest_df['locale'] = test_df.locale.fillna('None')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:06.654467Z","iopub.execute_input":"2025-01-07T12:20:06.654856Z","iopub.status.idle":"2025-01-07T12:20:09.622807Z","shell.execute_reply.started":"2025-01-07T12:20:06.654802Z","shell.execute_reply":"2025-01-07T12:20:09.621940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:09.623728Z","iopub.execute_input":"2025-01-07T12:20:09.624023Z","iopub.status.idle":"2025-01-07T12:20:15.733480Z","shell.execute_reply.started":"2025-01-07T12:20:09.623997Z","shell.execute_reply":"2025-01-07T12:20:15.732315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's encode holiday type in the following way\n\n1. Actual holidays will be labeled with 2\n2. Events will be labeled with 1, since they are\n3. Non holidays will be marekd with 0","metadata":{}},{"cell_type":"code","source":"df['holiday_type'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:15.734510Z","iopub.execute_input":"2025-01-07T12:20:15.734776Z","iopub.status.idle":"2025-01-07T12:20:16.757945Z","shell.execute_reply.started":"2025-01-07T12:20:15.734754Z","shell.execute_reply":"2025-01-07T12:20:16.757079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_type_mapping = {\"Transfer\": 2, \"Holiday\": 2, \"Additional\": 2, \"Event\": 1, \"Not holiday\": 0}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:16.758825Z","iopub.execute_input":"2025-01-07T12:20:16.759134Z","iopub.status.idle":"2025-01-07T12:20:16.763564Z","shell.execute_reply.started":"2025-01-07T12:20:16.759105Z","shell.execute_reply":"2025-01-07T12:20:16.762578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['holiday_type_encoded'] = df['holiday_type'].map(holiday_type_mapping)\ntest_df['holiday_type_encoded'] = test_df['holiday_type'].map(holiday_type_mapping)\n\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:16.764473Z","iopub.execute_input":"2025-01-07T12:20:16.764841Z","iopub.status.idle":"2025-01-07T12:20:18.096986Z","shell.execute_reply.started":"2025-01-07T12:20:16.764804Z","shell.execute_reply":"2025-01-07T12:20:18.095905Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's encode holiday locale in follwoing way (since we can assume order in holiday locale)\n\n1. None holiday will be labeled with 0\n2. Local holidays will be labeled with 1\n3. Regional holiday will be labeled with 2\n4. National - with 3","metadata":{}},{"cell_type":"code","source":"df.locale.unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:18.097964Z","iopub.execute_input":"2025-01-07T12:20:18.098243Z","iopub.status.idle":"2025-01-07T12:20:19.020073Z","shell.execute_reply.started":"2025-01-07T12:20:18.098220Z","shell.execute_reply":"2025-01-07T12:20:19.018996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_locale_mapping = {\"National\": 3, \"Regional\": 2, \"Local\": 1, \"None\": 0}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:19.021236Z","iopub.execute_input":"2025-01-07T12:20:19.021655Z","iopub.status.idle":"2025-01-07T12:20:19.026037Z","shell.execute_reply.started":"2025-01-07T12:20:19.021618Z","shell.execute_reply":"2025-01-07T12:20:19.025143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['holiday_locale_encoded'] = df['locale'].map(holiday_type_mapping)\ntest_df['holiday_locale_encoded'] = test_df['locale'].map(holiday_type_mapping)\n\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:19.026938Z","iopub.execute_input":"2025-01-07T12:20:19.027195Z","iopub.status.idle":"2025-01-07T12:20:21.359157Z","shell.execute_reply.started":"2025-01-07T12:20:19.027160Z","shell.execute_reply":"2025-01-07T12:20:21.358214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['onpromotion'] = df['onpromotion'].map({\"False\": 0, \"True\": 1})\ntest_df['onpromotion'] = test_df['onpromotion'].map({\"False\": 0, \"True\": 1})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:21.360123Z","iopub.execute_input":"2025-01-07T12:20:21.360510Z","iopub.status.idle":"2025-01-07T12:20:22.749457Z","shell.execute_reply.started":"2025-01-07T12:20:21.360470Z","shell.execute_reply":"2025-01-07T12:20:22.748672Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:20:22.750326Z","iopub.execute_input":"2025-01-07T12:20:22.750623Z","iopub.status.idle":"2025-01-07T12:20:30.059383Z","shell.execute_reply.started":"2025-01-07T12:20:22.750597Z","shell.execute_reply":"2025-01-07T12:20:30.058334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For other features we will create binary variables","metadata":{}},{"cell_type":"code","source":"ONE_HOT_FEATURES = ['family', 'city', 'state', 'store_type', 'cluster']\nNUMERICAL_FEATURES = ['oil_price']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:52:43.499770Z","iopub.execute_input":"2025-01-07T12:52:43.500151Z","iopub.status.idle":"2025-01-07T12:52:43.505260Z","shell.execute_reply.started":"2025-01-07T12:52:43.500122Z","shell.execute_reply":"2025-01-07T12:52:43.503928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.impute import SimpleImputer\n\npreprocessor = ColumnTransformer(\n    [\n        ('scaler', StandardScaler(), NUMERICAL_FEATURES),\n        ('one_hot', OneHotEncoder(handle_unknown='ignore'), ONE_HOT_FEATURES),\n        ('imputer', SimpleImputer(strategy=\"median\"), \n         ['onpromotion', 'is_holiday', 'perishable', 'holiday_type_encoded', 'holiday_locale_encoded']\n        )\n    ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:06:14.862965Z","iopub.execute_input":"2025-01-07T13:06:14.863323Z","iopub.status.idle":"2025-01-07T13:06:14.869336Z","shell.execute_reply.started":"2025-01-07T13:06:14.863295Z","shell.execute_reply":"2025-01-07T13:06:14.867995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Columns that will be used during training\n\nX_train = df[[\n    'onpromotion', 'is_holiday', 'perishable', \n    'oil_price', 'holiday_type_encoded', 'holiday_locale_encoded',\n    'family', 'city', 'state', 'store_type', 'cluster'\n]]\nX_test = test_df[[\n    'onpromotion', 'is_holiday', 'perishable', \n    'oil_price', 'holiday_type_encoded', 'holiday_locale_encoded',\n    'family', 'city', 'state', 'store_type', 'cluster'\n]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:06:16.373927Z","iopub.execute_input":"2025-01-07T13:06:16.374333Z","iopub.status.idle":"2025-01-07T13:06:18.175964Z","shell.execute_reply.started":"2025-01-07T13:06:16.374297Z","shell.execute_reply":"2025-01-07T13:06:18.175032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_preprocessed = preprocessor.fit_transform(X_train)\nX_test_preprocessed = preprocessor.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:06:18.177255Z","iopub.execute_input":"2025-01-07T13:06:18.177618Z","iopub.status.idle":"2025-01-07T13:06:59.178037Z","shell.execute_reply.started":"2025-01-07T13:06:18.177588Z","shell.execute_reply":"2025-01-07T13:06:59.176751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_preprocessed.shape, X_test_preprocessed.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:06:59.179970Z","iopub.execute_input":"2025-01-07T13:06:59.180395Z","iopub.status.idle":"2025-01-07T13:06:59.187078Z","shell.execute_reply.started":"2025-01-07T13:06:59.180349Z","shell.execute_reply":"2025-01-07T13:06:59.185856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train = df['unit_sales'].values\ny_test = test_df['unit_sales'].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:06:59.188365Z","iopub.execute_input":"2025-01-07T13:06:59.188730Z","iopub.status.idle":"2025-01-07T13:06:59.202810Z","shell.execute_reply.started":"2025-01-07T13:06:59.188684Z","shell.execute_reply":"2025-01-07T13:06:59.201609Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**3. Model training and optimization**","metadata":{}},{"cell_type":"markdown","source":"Let's start with simple linear regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nsimple_regression = LinearRegression()\nsimple_regression = simple_regression.fit(X_train_preprocessed, y_train)\n\nsimple_regression_prediction = simple_regression.predict(X_test_preprocessed)\n\nsimple_regression_score = nwrmsle(y_pred=simple_regression_prediction, y_true=y_test, weights=weights)\nsimple_regression_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T14:01:06.554350Z","iopub.execute_input":"2025-01-07T14:01:06.554779Z","iopub.status.idle":"2025-01-07T14:03:13.224202Z","shell.execute_reply.started":"2025-01-07T14:01:06.554746Z","shell.execute_reply":"2025-01-07T14:03:13.223072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\n\nridge = Ridge(alpha=1.0)\nridge = ridge.fit(X_train_preprocessed, y_train)\n\nridge_prediction = ridge.predict(X_test_preprocessed)\n\nridge_score = nwrmsle(y_pred=ridge_prediction, y_true=y_test, weights=weights)\nridge_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:11:04.351205Z","iopub.execute_input":"2025-01-07T13:11:04.351684Z","iopub.status.idle":"2025-01-07T13:12:39.930493Z","shell.execute_reply.started":"2025-01-07T13:11:04.351648Z","shell.execute_reply":"2025-01-07T13:12:39.929489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train_sp, X_val_sp, y_train_sp, y_val_sp = train_test_split(\n    X_train_preprocessed, y_train, test_size=0.15, random_state=42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:19:36.137238Z","iopub.execute_input":"2025-01-07T13:19:36.137825Z","iopub.status.idle":"2025-01-07T13:19:43.452637Z","shell.execute_reply.started":"2025-01-07T13:19:36.137763Z","shell.execute_reply":"2025-01-07T13:19:43.451349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\n\ntrain_data = lgb.Dataset(X_train_sp, label=y_train_sp)\nval_data = lgb.Dataset(X_val_sp, label=y_val_sp, reference=train_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:19:46.656853Z","iopub.execute_input":"2025-01-07T13:19:46.657191Z","iopub.status.idle":"2025-01-07T13:19:50.123737Z","shell.execute_reply.started":"2025-01-07T13:19:46.657164Z","shell.execute_reply":"2025-01-07T13:19:50.122536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    'num_leaves': 80,\n    'objective': 'regression',\n    'min_data_in_leaf': 200,\n    'learning_rate': 0.02,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.7,\n    'bagging_freq': 1,\n    'metric': 'l2',\n    'num_threads': 16\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T13:40:32.630985Z","iopub.execute_input":"2025-01-07T13:40:32.631516Z","iopub.status.idle":"2025-01-07T13:40:32.637803Z","shell.execute_reply.started":"2025-01-07T13:40:32.631470Z","shell.execute_reply":"2025-01-07T13:40:32.636571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result = lgb.train(\n    params, \n    train_data, \n    num_boost_round=500,\n    valid_sets=[train_data, val_data]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T14:06:21.594206Z","iopub.execute_input":"2025-01-07T14:06:21.594631Z","iopub.status.idle":"2025-01-07T14:15:31.847828Z","shell.execute_reply.started":"2025-01-07T14:06:21.594586Z","shell.execute_reply":"2025-01-07T14:15:31.846688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_prediction = result.predict(X_test_preprocessed)\nlgbm_score = nwrmsle(y_pred=lgbm_prediction, y_true=y_test, weights=weights)\nlgbm_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T14:15:31.849081Z","iopub.execute_input":"2025-01-07T14:15:31.849346Z","iopub.status.idle":"2025-01-07T14:16:37.638795Z","shell.execute_reply.started":"2025-01-07T14:15:31.849323Z","shell.execute_reply":"2025-01-07T14:16:37.637873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}