{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"cdfe1563-be58-4335-bc8f-9d9af031d72d","_cell_guid":"e9b0a329-d35c-4f2c-aa5b-c8076f85b306","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:24:58.112354Z","iopub.execute_input":"2025-01-14T17:24:58.112958Z","iopub.status.idle":"2025-01-14T17:24:59.338584Z","shell.execute_reply.started":"2025-01-14T17:24:58.112920Z","shell.execute_reply":"2025-01-14T17:24:59.337403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        paths.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"5dc5203c-1772-4020-83bc-7cebcc19180f","_cell_guid":"7f048c52-40dd-4e54-b1a1-a832c85a67c4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:24:59.347921Z","iopub.execute_input":"2025-01-14T17:24:59.348272Z","iopub.status.idle":"2025-01-14T17:24:59.357827Z","shell.execute_reply.started":"2025-01-14T17:24:59.348236Z","shell.execute_reply":"2025-01-14T17:24:59.356515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"_uuid":"546a4749-f5f7-4533-8c63-c0e0ff97e4f7","_cell_guid":"ef1d1e2c-7f13-4a5d-9983-b7533bba2d30","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:24:59.361782Z","iopub.execute_input":"2025-01-14T17:24:59.362139Z","iopub.status.idle":"2025-01-14T17:25:04.328193Z","shell.execute_reply.started":"2025-01-14T17:24:59.362102Z","shell.execute_reply":"2025-01-14T17:25:04.326962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfor filenames in paths:\n    with py7zr.SevenZipFile(filenames, mode='r') as z_ref:\n        z_ref.extractall(path='/kaggle/working')","metadata":{"_uuid":"bc064aa4-9475-471e-8712-ba529fe2610c","_cell_guid":"6a132473-e7a9-472f-8624-ab6af5540197","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:25:04.329487Z","iopub.execute_input":"2025-01-14T17:25:04.329925Z","iopub.status.idle":"2025-01-14T17:26:33.527471Z","shell.execute_reply.started":"2025-01-14T17:25:04.329885Z","shell.execute_reply":"2025-01-14T17:26:33.526156Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{"_uuid":"f8122754-f8ce-4b4f-af0c-6a7d5b9a4205","_cell_guid":"66e6e384-9baa-488c-bb26-1613c1b45e07","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_data.csv`** – the test dataset used to evaluate the model's performance.","metadata":{"_uuid":"7ceb4af9-18ac-4031-ace0-f7ce92f98bee","_cell_guid":"26b0b96a-94ae-4bfa-ad1b-101a039e4381","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"%%time\ndata = pd.read_csv(\"/kaggle/working/train.csv\")\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"_uuid":"a8451afb-473e-4f62-a1e0-050bb2463df7","_cell_guid":"c28627f8-2632-4d8d-874e-f996a1e5c789","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:26:33.528751Z","iopub.execute_input":"2025-01-14T17:26:33.529041Z","iopub.status.idle":"2025-01-14T17:28:35.159830Z","shell.execute_reply.started":"2025-01-14T17:26:33.529019Z","shell.execute_reply":"2025-01-14T17:28:35.158756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"_uuid":"631eba1e-f080-416e-8a7f-e99aabc50ac0","_cell_guid":"409d1cbd-e934-453f-8ee8-0c4a9f7dfc59","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:28:35.163105Z","iopub.execute_input":"2025-01-14T17:28:35.163409Z","iopub.status.idle":"2025-01-14T17:28:42.621792Z","shell.execute_reply.started":"2025-01-14T17:28:35.163385Z","shell.execute_reply":"2025-01-14T17:28:42.620684Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Since RAM in kaggle does not allow porcess the whole data we will set lower threshold value for our train to decrease the amount of train data. We will train on data since 2016.07.01","metadata":{}},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE_LOW = pd.to_datetime(\"2016-07-01\")\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE_LOW)&(data[\"date\"] < THRESHOLD_TRAIN_DATE)].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"_uuid":"4c9ec827-cada-48a5-93ca-743b05c136de","_cell_guid":"1844559e-0f58-4815-a4d7-0f5c682603fb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:28:42.623822Z","iopub.execute_input":"2025-01-14T17:28:42.624357Z","iopub.status.idle":"2025-01-14T17:31:05.480181Z","shell.execute_reply.started":"2025-01-14T17:28:42.624326Z","shell.execute_reply":"2025-01-14T17:31:05.479054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%reset -f","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:05.481369Z","iopub.execute_input":"2025-01-14T17:31:05.481741Z","iopub.status.idle":"2025-01-14T17:31:05.682843Z","shell.execute_reply.started":"2025-01-14T17:31:05.481712Z","shell.execute_reply":"2025-01-14T17:31:05.681537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:05.683998Z","iopub.execute_input":"2025-01-14T17:31:05.684461Z","iopub.status.idle":"2025-01-14T17:31:05.698896Z","shell.execute_reply.started":"2025-01-14T17:31:05.684414Z","shell.execute_reply":"2025-01-14T17:31:05.697784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\"/kaggle/working/train_data.csv\")\ntest_df = pd.read_csv(\"/kaggle/working/test_data.csv\")","metadata":{"_uuid":"d5f793e7-1316-4bda-93ea-d3236ed62162","_cell_guid":"45dfd650-9ade-4946-95e1-c1ef947fec23","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:05.700203Z","iopub.execute_input":"2025-01-14T17:31:05.700586Z","iopub.status.idle":"2025-01-14T17:31:32.938240Z","shell.execute_reply.started":"2025-01-14T17:31:05.700538Z","shell.execute_reply":"2025-01-14T17:31:32.937102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"date\"] = pd.to_datetime(df[\"date\"])\ntest_df[\"date\"] = pd.to_datetime(test_df[\"date\"])","metadata":{"_uuid":"9e5b42df-0cd7-4699-9452-985dad267d61","_cell_guid":"67f95d3f-aecc-473b-be48-4f748508521c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:32.939535Z","iopub.execute_input":"2025-01-14T17:31:32.939961Z","iopub.status.idle":"2025-01-14T17:31:37.814050Z","shell.execute_reply.started":"2025-01-14T17:31:32.939923Z","shell.execute_reply":"2025-01-14T17:31:37.813243Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"_uuid":"ad55ebf6-0d43-46c4-ab93-62831fa9d164","_cell_guid":"4c3837e9-ce70-4e8e-891b-0bd1e5109ecd","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"_uuid":"dfd4f70b-edd3-407b-bd36-90abe7224467","_cell_guid":"cb558e17-7f9d-4e81-a04d-dc6930d65a26","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:37.815188Z","iopub.execute_input":"2025-01-14T17:31:37.815559Z","iopub.status.idle":"2025-01-14T17:31:39.942001Z","shell.execute_reply.started":"2025-01-14T17:31:37.815514Z","shell.execute_reply":"2025-01-14T17:31:39.940848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"_uuid":"fb08c9c6-b201-40c8-a2e1-4d5a6b6a0cec","_cell_guid":"222c075e-58bd-4b9d-9f11-6e778d573d55","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:39.943193Z","iopub.execute_input":"2025-01-14T17:31:39.943597Z","iopub.status.idle":"2025-01-14T17:31:40.433392Z","shell.execute_reply.started":"2025-01-14T17:31:39.943556Z","shell.execute_reply":"2025-01-14T17:31:40.432367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"3b96526c-9b8d-4baa-bb46-bbec415b1c51","_cell_guid":"bec52c80-d3cb-4378-add5-230a335b7a42","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.434068Z","iopub.execute_input":"2025-01-14T17:31:40.434329Z","iopub.status.idle":"2025-01-14T17:31:40.450300Z","shell.execute_reply.started":"2025-01-14T17:31:40.434309Z","shell.execute_reply":"2025-01-14T17:31:40.449180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:40.451454Z","iopub.execute_input":"2025-01-14T17:31:40.451962Z","iopub.status.idle":"2025-01-14T17:31:40.457786Z","shell.execute_reply.started":"2025-01-14T17:31:40.451924Z","shell.execute_reply":"2025-01-14T17:31:40.456913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:40.458685Z","iopub.execute_input":"2025-01-14T17:31:40.459025Z","iopub.status.idle":"2025-01-14T17:31:40.474666Z","shell.execute_reply.started":"2025-01-14T17:31:40.458999Z","shell.execute_reply":"2025-01-14T17:31:40.473524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"_uuid":"c78ca284-383d-4170-b58f-e42d40978a26","_cell_guid":"a01937f9-b78e-4207-bcc2-1f6b46d3d383","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z"},"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"_uuid":"2243939f-19a7-4121-b146-1e3b8a0a9aaa","_cell_guid":"6a24950d-65de-4815-a935-e0abb7552c7d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.476066Z","iopub.execute_input":"2025-01-14T17:31:40.476466Z","iopub.status.idle":"2025-01-14T17:31:40.500142Z","shell.execute_reply.started":"2025-01-14T17:31:40.476426Z","shell.execute_reply":"2025-01-14T17:31:40.499068Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{"_uuid":"0a1f4ca2-119b-433b-af18-e6e42eeabc41","_cell_guid":"45291d9a-a5fe-4ef8-927f-41a9781d087a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\nprint(items.shape)\nitems.head()","metadata":{"_uuid":"ec2ad2d3-23e1-4a49-a85a-41f4e2a8f6a6","_cell_guid":"d895ffaf-772c-4351-b5b2-21e3473be84d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.501471Z","iopub.execute_input":"2025-01-14T17:31:40.501974Z","iopub.status.idle":"2025-01-14T17:31:40.516116Z","shell.execute_reply.started":"2025-01-14T17:31:40.501945Z","shell.execute_reply":"2025-01-14T17:31:40.515204Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{"_uuid":"84712e5e-cbe5-492e-aff7-60cd2639db2c","_cell_guid":"cb9c9dbc-072b-4048-9485-08cb7c64256e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\nprint(oil.shape)\noil.head()","metadata":{"_uuid":"e222e3c2-23f8-4643-ad40-8a21b3abdcde","_cell_guid":"1be7db10-c731-47ba-b676-dfca419211ec","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.517169Z","iopub.execute_input":"2025-01-14T17:31:40.517571Z","iopub.status.idle":"2025-01-14T17:31:40.529611Z","shell.execute_reply.started":"2025-01-14T17:31:40.517531Z","shell.execute_reply":"2025-01-14T17:31:40.528713Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.","metadata":{"_uuid":"a6b78abe-4094-48c7-b40d-ca04edaed72c","_cell_guid":"d3e57a71-e064-40e5-9ac5-93a9724195e1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()","metadata":{"_uuid":"6e85793d-d81b-4eb1-908b-6eaa7792a758","_cell_guid":"5838202d-69cb-425b-afff-33bca5147a40","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.530596Z","iopub.execute_input":"2025-01-14T17:31:40.530969Z","iopub.status.idle":"2025-01-14T17:31:40.550840Z","shell.execute_reply.started":"2025-01-14T17:31:40.530932Z","shell.execute_reply":"2025-01-14T17:31:40.549952Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)","metadata":{"_uuid":"ace4e9a3-aef7-40f0-8810-e6f909dcc930","_cell_guid":"5f230e3f-1ee2-4fd5-acad-3f3cdb8b9288","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:40.551898Z","iopub.execute_input":"2025-01-14T17:31:40.552230Z","iopub.status.idle":"2025-01-14T17:31:40.566706Z","shell.execute_reply.started":"2025-01-14T17:31:40.552192Z","shell.execute_reply":"2025-01-14T17:31:40.565727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"efdefbce-5f73-4454-8db7-732730b6c0cd","_cell_guid":"19107833-946b-4974-bdc8-147253fdf7c7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.567711Z","iopub.execute_input":"2025-01-14T17:31:40.568095Z","iopub.status.idle":"2025-01-14T17:31:40.587698Z","shell.execute_reply.started":"2025-01-14T17:31:40.568056Z","shell.execute_reply":"2025-01-14T17:31:40.586506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum() / len(df)","metadata":{"_uuid":"2f121665-92bd-4a32-905f-8e4f233002e7","_cell_guid":"ed4bf518-25e3-4b91-9fab-fc87bab31313","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.588886Z","iopub.execute_input":"2025-01-14T17:31:40.589258Z","iopub.status.idle":"2025-01-14T17:31:40.857300Z","shell.execute_reply.started":"2025-01-14T17:31:40.589218Z","shell.execute_reply":"2025-01-14T17:31:40.856475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['onpromotion'].value_counts()","metadata":{"_uuid":"fa89ad47-3f15-4285-ba5e-d22c58567301","_cell_guid":"61fa79df-6216-45e0-99e5-9c5c9f025a9c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-14T17:31:40.862933Z","iopub.execute_input":"2025-01-14T17:31:40.863226Z","iopub.status.idle":"2025-01-14T17:31:41.099301Z","shell.execute_reply.started":"2025-01-14T17:31:40.863204Z","shell.execute_reply":"2025-01-14T17:31:41.098300Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As can be seen, 17% of data in onpromotion column is missing. And most of the data has false on this column. Since 17% is considerable amount of data, we will create new column for missing values.","metadata":{}},{"cell_type":"code","source":"df['onpromotion'] = df['onpromotion'].fillna('unknown')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:41.102784Z","iopub.execute_input":"2025-01-14T17:31:41.103197Z","iopub.status.idle":"2025-01-14T17:31:41.126786Z","shell.execute_reply.started":"2025-01-14T17:31:41.103157Z","shell.execute_reply":"2025-01-14T17:31:41.125967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['onpromotion'] = test_df['onpromotion'].fillna('unknown')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:41.127978Z","iopub.execute_input":"2025-01-14T17:31:41.128349Z","iopub.status.idle":"2025-01-14T17:31:41.134216Z","shell.execute_reply.started":"2025-01-14T17:31:41.128316Z","shell.execute_reply":"2025-01-14T17:31:41.133249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['item_nbr'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:41.135213Z","iopub.execute_input":"2025-01-14T17:31:41.135640Z","iopub.status.idle":"2025-01-14T17:31:43.372124Z","shell.execute_reply.started":"2025-01-14T17:31:41.135579Z","shell.execute_reply":"2025-01-14T17:31:43.371047Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Since there are too many items to categorize them and make model to distinguish each item, we will not condiser each item as indiviudal object but train on the feature of each item from items table","metadata":{}},{"cell_type":"code","source":"items","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:43.372907Z","iopub.execute_input":"2025-01-14T17:31:43.373193Z","iopub.status.idle":"2025-01-14T17:31:43.391333Z","shell.execute_reply.started":"2025-01-14T17:31:43.373157Z","shell.execute_reply":"2025-01-14T17:31:43.390332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"items.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:43.392335Z","iopub.execute_input":"2025-01-14T17:31:43.392614Z","iopub.status.idle":"2025-01-14T17:31:43.412081Z","shell.execute_reply.started":"2025-01-14T17:31:43.392584Z","shell.execute_reply":"2025-01-14T17:31:43.411011Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Family and perishable will be categoriacl features and class continuous.","metadata":{}},{"cell_type":"code","source":"items['family'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:43.413140Z","iopub.execute_input":"2025-01-14T17:31:43.413447Z","iopub.status.idle":"2025-01-14T17:31:43.429839Z","shell.execute_reply.started":"2025-01-14T17:31:43.413421Z","shell.execute_reply":"2025-01-14T17:31:43.428969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"items.isna().sum()/len(items)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:43.431065Z","iopub.execute_input":"2025-01-14T17:31:43.431462Z","iopub.status.idle":"2025-01-14T17:31:43.447765Z","shell.execute_reply.started":"2025-01-14T17:31:43.431423Z","shell.execute_reply":"2025-01-14T17:31:43.446860Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Items has no missing values and will merge it to our dataset","metadata":{}},{"cell_type":"code","source":"df = df.merge(items, on=\"item_nbr\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:43.448775Z","iopub.execute_input":"2025-01-14T17:31:43.449114Z","iopub.status.idle":"2025-01-14T17:31:50.362429Z","shell.execute_reply.started":"2025-01-14T17:31:43.449083Z","shell.execute_reply":"2025-01-14T17:31:50.361456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:50.363601Z","iopub.execute_input":"2025-01-14T17:31:50.364018Z","iopub.status.idle":"2025-01-14T17:31:50.380485Z","shell.execute_reply.started":"2025-01-14T17:31:50.363979Z","shell.execute_reply":"2025-01-14T17:31:50.379383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = test_df.merge(items, on=\"item_nbr\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:50.381498Z","iopub.execute_input":"2025-01-14T17:31:50.381836Z","iopub.status.idle":"2025-01-14T17:31:50.916299Z","shell.execute_reply.started":"2025-01-14T17:31:50.381811Z","shell.execute_reply":"2025-01-14T17:31:50.915452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum()/len(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:50.917219Z","iopub.execute_input":"2025-01-14T17:31:50.917531Z","iopub.status.idle":"2025-01-14T17:31:52.909031Z","shell.execute_reply.started":"2025-01-14T17:31:50.917506Z","shell.execute_reply":"2025-01-14T17:31:52.908073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['store_nbr'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:52.910021Z","iopub.execute_input":"2025-01-14T17:31:52.910356Z","iopub.status.idle":"2025-01-14T17:31:53.124641Z","shell.execute_reply.started":"2025-01-14T17:31:52.910323Z","shell.execute_reply":"2025-01-14T17:31:53.123616Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Similarly, we extract features of stores","metadata":{}},{"cell_type":"code","source":"stores.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:53.125638Z","iopub.execute_input":"2025-01-14T17:31:53.125948Z","iopub.status.idle":"2025-01-14T17:31:53.145241Z","shell.execute_reply.started":"2025-01-14T17:31:53.125924Z","shell.execute_reply":"2025-01-14T17:31:53.144176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores['cluster'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:53.146275Z","iopub.execute_input":"2025-01-14T17:31:53.146609Z","iopub.status.idle":"2025-01-14T17:31:53.162136Z","shell.execute_reply.started":"2025-01-14T17:31:53.146567Z","shell.execute_reply":"2025-01-14T17:31:53.161127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(stores, on=\"store_nbr\", how=\"left\")\ntest_df = test_df.merge(stores, on=\"store_nbr\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:31:53.163148Z","iopub.execute_input":"2025-01-14T17:31:53.163490Z","iopub.status.idle":"2025-01-14T17:32:07.369886Z","shell.execute_reply.started":"2025-01-14T17:31:53.163465Z","shell.execute_reply":"2025-01-14T17:32:07.368577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:07.371251Z","iopub.execute_input":"2025-01-14T17:32:07.371643Z","iopub.status.idle":"2025-01-14T17:32:07.388212Z","shell.execute_reply.started":"2025-01-14T17:32:07.371585Z","shell.execute_reply":"2025-01-14T17:32:07.387300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum()/len(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:07.389277Z","iopub.execute_input":"2025-01-14T17:32:07.389561Z","iopub.status.idle":"2025-01-14T17:32:14.191024Z","shell.execute_reply.started":"2025-01-14T17:32:07.389536Z","shell.execute_reply":"2025-01-14T17:32:14.189859Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now merge oil prices","metadata":{}},{"cell_type":"code","source":"oil['date'] = pd.to_datetime(oil['date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.192082Z","iopub.execute_input":"2025-01-14T17:32:14.192387Z","iopub.status.idle":"2025-01-14T17:32:14.198782Z","shell.execute_reply.started":"2025-01-14T17:32:14.192353Z","shell.execute_reply":"2025-01-14T17:32:14.197823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.199724Z","iopub.execute_input":"2025-01-14T17:32:14.200051Z","iopub.status.idle":"2025-01-14T17:32:14.220246Z","shell.execute_reply.started":"2025-01-14T17:32:14.200011Z","shell.execute_reply":"2025-01-14T17:32:14.219358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil.isna().sum()/len(oil)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.221215Z","iopub.execute_input":"2025-01-14T17:32:14.221537Z","iopub.status.idle":"2025-01-14T17:32:14.240832Z","shell.execute_reply.started":"2025-01-14T17:32:14.221511Z","shell.execute_reply":"2025-01-14T17:32:14.239842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.241991Z","iopub.execute_input":"2025-01-14T17:32:14.242423Z","iopub.status.idle":"2025-01-14T17:32:14.265987Z","shell.execute_reply.started":"2025-01-14T17:32:14.242381Z","shell.execute_reply":"2025-01-14T17:32:14.264945Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Oil value has missing values on both dates and price. It is better to consider average oil price for each month to handle this.","metadata":{}},{"cell_type":"code","source":"oil[\"year\"] = oil[\"date\"].dt.year\noil[\"month\"] = oil[\"date\"].dt.month\n\nmonthly_oil = (\n    oil.groupby([\"year\", \"month\"])[\"dcoilwtico\"]\n    .mean()\n    .reset_index()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.267043Z","iopub.execute_input":"2025-01-14T17:32:14.267341Z","iopub.status.idle":"2025-01-14T17:32:14.285213Z","shell.execute_reply.started":"2025-01-14T17:32:14.267317Z","shell.execute_reply":"2025-01-14T17:32:14.284171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_oil['month-year'] = pd.to_datetime(monthly_oil.year.astype(str) + '/' + monthly_oil.month.astype(str) + '/01')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.286276Z","iopub.execute_input":"2025-01-14T17:32:14.286536Z","iopub.status.idle":"2025-01-14T17:32:14.302804Z","shell.execute_reply.started":"2025-01-14T17:32:14.286516Z","shell.execute_reply":"2025-01-14T17:32:14.301739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nsns.lineplot(data=monthly_oil, y='dcoilwtico', x='month-year')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.303907Z","iopub.execute_input":"2025-01-14T17:32:14.304178Z","iopub.status.idle":"2025-01-14T17:32:14.636605Z","shell.execute_reply.started":"2025-01-14T17:32:14.304155Z","shell.execute_reply":"2025-01-14T17:32:14.635590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_oil[monthly_oil['year'] >= 2016]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.637572Z","iopub.execute_input":"2025-01-14T17:32:14.637921Z","iopub.status.idle":"2025-01-14T17:32:14.650091Z","shell.execute_reply.started":"2025-01-14T17:32:14.637894Z","shell.execute_reply":"2025-01-14T17:32:14.649071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"year\"] = df[\"date\"].dt.year\ndf[\"month\"] = df[\"date\"].dt.month\ntest_df[\"year\"] = test_df[\"date\"].dt.year\ntest_df[\"month\"] = test_df[\"date\"].dt.month","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:14.651012Z","iopub.execute_input":"2025-01-14T17:32:14.651398Z","iopub.status.idle":"2025-01-14T17:32:16.677175Z","shell.execute_reply.started":"2025-01-14T17:32:14.651368Z","shell.execute_reply":"2025-01-14T17:32:16.676013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(monthly_oil, on=[\"year\", \"month\"], how=\"left\")\ntest_df = test_df.merge(monthly_oil, on=[\"year\", \"month\"], how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:16.678133Z","iopub.execute_input":"2025-01-14T17:32:16.678522Z","iopub.status.idle":"2025-01-14T17:32:34.439806Z","shell.execute_reply.started":"2025-01-14T17:32:16.678484Z","shell.execute_reply":"2025-01-14T17:32:34.438764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:34.440879Z","iopub.execute_input":"2025-01-14T17:32:34.441164Z","iopub.status.idle":"2025-01-14T17:32:34.458443Z","shell.execute_reply.started":"2025-01-14T17:32:34.441141Z","shell.execute_reply":"2025-01-14T17:32:34.457526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum() / len(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:34.459545Z","iopub.execute_input":"2025-01-14T17:32:34.459989Z","iopub.status.idle":"2025-01-14T17:32:41.849757Z","shell.execute_reply.started":"2025-01-14T17:32:34.459960Z","shell.execute_reply":"2025-01-14T17:32:41.848704Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now we will add holidays","metadata":{}},{"cell_type":"code","source":"holiday_events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:41.850926Z","iopub.execute_input":"2025-01-14T17:32:41.851308Z","iopub.status.idle":"2025-01-14T17:32:41.864919Z","shell.execute_reply.started":"2025-01-14T17:32:41.851272Z","shell.execute_reply":"2025-01-14T17:32:41.863820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_events['date'] = pd.to_datetime(holiday_events['date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:41.866209Z","iopub.execute_input":"2025-01-14T17:32:41.866530Z","iopub.status.idle":"2025-01-14T17:32:41.885467Z","shell.execute_reply.started":"2025-01-14T17:32:41.866498Z","shell.execute_reply":"2025-01-14T17:32:41.884403Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's keep only national holidays since it easier to process","metadata":{}},{"cell_type":"code","source":"holiday_events[\"is_holiday\"] = 1\nholidays_national = holiday_events[holiday_events[\"locale\"]==\"National\"][[\"date\",\"is_holiday\"]]\nholidays_national = holidays_national.drop_duplicates(\"date\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:41.886547Z","iopub.execute_input":"2025-01-14T17:32:41.886885Z","iopub.status.idle":"2025-01-14T17:32:41.906707Z","shell.execute_reply.started":"2025-01-14T17:32:41.886858Z","shell.execute_reply":"2025-01-14T17:32:41.905423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(holidays_national, on=\"date\", how=\"left\")\ntest_df = test_df.merge(holidays_national, on=\"date\", how=\"left\")\n\ndf[\"is_holiday\"] = df[\"is_holiday\"].fillna(0)\ntest_df[\"is_holiday\"] = test_df[\"is_holiday\"].fillna(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:41.908405Z","iopub.execute_input":"2025-01-14T17:32:41.908850Z","iopub.status.idle":"2025-01-14T17:32:51.290671Z","shell.execute_reply.started":"2025-01-14T17:32:41.908811Z","shell.execute_reply":"2025-01-14T17:32:51.289748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['is_holiday'] = (df['is_holiday'] == 1)\ntest_df[\"is_holiday\"] = (test_df[\"is_holiday\"] == 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:51.291578Z","iopub.execute_input":"2025-01-14T17:32:51.291971Z","iopub.status.idle":"2025-01-14T17:32:51.366009Z","shell.execute_reply.started":"2025-01-14T17:32:51.291937Z","shell.execute_reply":"2025-01-14T17:32:51.365112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['is_holiday'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:51.367013Z","iopub.execute_input":"2025-01-14T17:32:51.367282Z","iopub.status.idle":"2025-01-14T17:32:51.604224Z","shell.execute_reply.started":"2025-01-14T17:32:51.367259Z","shell.execute_reply":"2025-01-14T17:32:51.603335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum() / len(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:51.605412Z","iopub.execute_input":"2025-01-14T17:32:51.605755Z","iopub.status.idle":"2025-01-14T17:32:58.866353Z","shell.execute_reply.started":"2025-01-14T17:32:51.605710Z","shell.execute_reply":"2025-01-14T17:32:58.865395Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Also it will be helpful to indicate wheter the day is weekend","metadata":{}},{"cell_type":"code","source":"df['is_weekend'] = df['date'].dt.dayofweek >= 5\ntest_df['is_weekend'] = test_df['date'].dt.dayofweek >= 5\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:32:58.867395Z","iopub.execute_input":"2025-01-14T17:32:58.867808Z","iopub.status.idle":"2025-01-14T17:33:00.196460Z","shell.execute_reply.started":"2025-01-14T17:32:58.867778Z","shell.execute_reply":"2025-01-14T17:33:00.195349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:33:00.197347Z","iopub.execute_input":"2025-01-14T17:33:00.197660Z","iopub.status.idle":"2025-01-14T17:33:00.224612Z","shell.execute_reply.started":"2025-01-14T17:33:00.197616Z","shell.execute_reply":"2025-01-14T17:33:00.223540Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We will create columns transformer to handle numeric and categorical features","metadata":{}},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:33:00.225984Z","iopub.execute_input":"2025-01-14T17:33:00.226563Z","iopub.status.idle":"2025-01-14T17:33:00.233140Z","shell.execute_reply.started":"2025-01-14T17:33:00.226520Z","shell.execute_reply":"2025-01-14T17:33:00.232122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric = ['class', 'dcoilwtico', 'perishable']\ncategorical = ['onpromotion', 'family', 'class', 'type', 'cluster', 'is_holiday', 'is_weekend']\nlabel = 'unit_sales'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:33:00.234167Z","iopub.execute_input":"2025-01-14T17:33:00.234594Z","iopub.status.idle":"2025-01-14T17:33:00.247301Z","shell.execute_reply.started":"2025-01-14T17:33:00.234553Z","shell.execute_reply":"2025-01-14T17:33:00.246304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = df[numeric + categorical]\ny_train = df[label]\nX_test = test_df[numeric + categorical]\ny_test = test_df[label]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:33:00.248320Z","iopub.execute_input":"2025-01-14T17:33:00.248730Z","iopub.status.idle":"2025-01-14T17:33:02.046591Z","shell.execute_reply.started":"2025-01-14T17:33:00.248689Z","shell.execute_reply":"2025-01-14T17:33:02.045604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.to_csv(\"/kaggle/working/X_train.csv\")\ny_train.to_csv(\"/kaggle/working/y_train.csv\")\nX_test.to_csv(\"/kaggle/working/X_test.csv\")\ny_test.to_csv(\"/kaggle/working/y_test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:33:02.047769Z","iopub.execute_input":"2025-01-14T17:33:02.048182Z","iopub.status.idle":"2025-01-14T17:36:29.560285Z","shell.execute_reply.started":"2025-01-14T17:33:02.048142Z","shell.execute_reply":"2025-01-14T17:36:29.559100Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%reset -f","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:36:29.561382Z","iopub.execute_input":"2025-01-14T17:36:29.561714Z","iopub.status.idle":"2025-01-14T17:36:32.014482Z","shell.execute_reply.started":"2025-01-14T17:36:29.561687Z","shell.execute_reply":"2025-01-14T17:36:32.013326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:36:32.015529Z","iopub.execute_input":"2025-01-14T17:36:32.015857Z","iopub.status.idle":"2025-01-14T17:36:32.020486Z","shell.execute_reply.started":"2025-01-14T17:36:32.015830Z","shell.execute_reply":"2025-01-14T17:36:32.019292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = pd.read_csv(\"/kaggle/working/X_train.csv\")\ny_train = pd.read_csv(\"/kaggle/working/y_train.csv\")\nX_test = pd.read_csv(\"/kaggle/working/X_test.csv\")\ny_test = pd.read_csv(\"/kaggle/working/y_test.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:36:32.021421Z","iopub.execute_input":"2025-01-14T17:36:32.021774Z","iopub.status.idle":"2025-01-14T17:37:34.224796Z","shell.execute_reply.started":"2025-01-14T17:36:32.021734Z","shell.execute_reply":"2025-01-14T17:37:34.223555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train = y_train['unit_sales']\ny_test = y_test['unit_sales']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:34.226038Z","iopub.execute_input":"2025-01-14T17:37:34.226445Z","iopub.status.idle":"2025-01-14T17:37:34.232647Z","shell.execute_reply.started":"2025-01-14T17:37:34.226405Z","shell.execute_reply":"2025-01-14T17:37:34.231455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:34.233753Z","iopub.execute_input":"2025-01-14T17:37:34.234067Z","iopub.status.idle":"2025-01-14T17:37:34.264797Z","shell.execute_reply.started":"2025-01-14T17:37:34.234027Z","shell.execute_reply":"2025-01-14T17:37:34.263721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.model_selection import GridSearchCV\n\nimport lightgbm as lgb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:34.265962Z","iopub.execute_input":"2025-01-14T17:37:34.266362Z","iopub.status.idle":"2025-01-14T17:37:35.751408Z","shell.execute_reply.started":"2025-01-14T17:37:34.266326Z","shell.execute_reply":"2025-01-14T17:37:35.750297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric = ['class', 'dcoilwtico', 'perishable']\ncategorical = ['onpromotion', 'family', 'class', 'type', 'cluster', 'is_holiday', 'is_weekend']\nlabel = 'unit_sales'\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', StandardScaler(), numeric),\n        ('cat', OneHotEncoder(handle_unknown='ignore'), categorical)\n    ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:35.752523Z","iopub.execute_input":"2025-01-14T17:37:35.753435Z","iopub.status.idle":"2025-01-14T17:37:35.758565Z","shell.execute_reply.started":"2025-01-14T17:37:35.753390Z","shell.execute_reply":"2025-01-14T17:37:35.757739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalized_weighted_rmsle(y_true, y_pred, weights):\n    log_errors = np.log1p(y_pred) - np.log1p(y_true)\n    squared_log_errors = log_errors ** 2\n    weighted_squared_log_errors = weights * squared_log_errors\n    sum_weighted_errors = np.sum(weighted_squared_log_errors)\n    sum_weights = np.sum(weights)\n    normalized_weighted_mean = sum_weighted_errors / sum_weights\n    return np.sqrt(normalized_weighted_mean)\nweights_train = np.where(X_train['perishable'] == 1, 1.25, 1)\nweights_test = np.where(X_test['perishable'] == 1, 1.25, 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:35.759642Z","iopub.execute_input":"2025-01-14T17:37:35.760011Z","iopub.status.idle":"2025-01-14T17:37:35.991058Z","shell.execute_reply.started":"2025-01-14T17:37:35.759976Z","shell.execute_reply":"2025-01-14T17:37:35.989830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"min(y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:35.992206Z","iopub.execute_input":"2025-01-14T17:37:35.992613Z","iopub.status.idle":"2025-01-14T17:37:39.284545Z","shell.execute_reply.started":"2025-01-14T17:37:35.992582Z","shell.execute_reply":"2025-01-14T17:37:39.283629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.sum(y_train < 0) / len(y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:39.285604Z","iopub.execute_input":"2025-01-14T17:37:39.285997Z","iopub.status.idle":"2025-01-14T17:37:39.356603Z","shell.execute_reply.started":"2025-01-14T17:37:39.285963Z","shell.execute_reply":"2025-01-14T17:37:39.355466Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Since we have negatives for sales and use logarithms for metrics we should replace negatives with 0","metadata":{}},{"cell_type":"code","source":"y_test = np.maximum(y_test, 0)\ny_train = np.maximum(y_train, 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:39.357747Z","iopub.execute_input":"2025-01-14T17:37:39.358048Z","iopub.status.idle":"2025-01-14T17:37:39.473536Z","shell.execute_reply.started":"2025-01-14T17:37:39.358024Z","shell.execute_reply":"2025-01-14T17:37:39.472410Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"markdown","source":"Since it is regreession task, we will use dummyregressor for baseline model","metadata":{}},{"cell_type":"code","source":"dummy_reg = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('regressor', DummyRegressor(strategy='mean'))\n])\ndummy_reg.fit(X_train, y_train)\n\ny_train_pred_dummy = dummy_reg.predict(X_train)\ny_test_pred_dummy = dummy_reg.predict(X_test)\n\nwrmsle_train_dummy = normalized_weighted_rmsle(y_train, y_train_pred_dummy, weights_train)\nwrmsle_test_dummy = normalized_weighted_rmsle(y_test, y_test_pred_dummy, weights_test)\n\n\nprint(f\"Train Normalized Weighted RMSLE: {wrmsle_train_dummy}, Test Normalized Weighted RMSLE: {wrmsle_test_dummy}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:37:39.480868Z","iopub.execute_input":"2025-01-14T17:37:39.481196Z","iopub.status.idle":"2025-01-14T17:39:26.818667Z","shell.execute_reply.started":"2025-01-14T17:37:39.481169Z","shell.execute_reply":"2025-01-14T17:39:26.817720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_pipeline = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('regressor', lgb.LGBMRegressor(objective='regression', boosting_type='gbdt', learning_rate=0.1, \n                                    num_leaves=30, n_estimators=200, random_state=42, n_jobs=-1))\n])\n\nlgb_pipeline.fit(X_train, y_train)\n\ny_train_pred_lgb = lgb_pipeline.predict(X_train)\ny_test_pred_lgb = lgb_pipeline.predict(X_test)\n\nwrmsle_train_lgb = normalized_weighted_rmsle(y_train, y_train_pred_lgb, weights_train)\nwrmsle_test_lgb = normalized_weighted_rmsle(y_test, y_test_pred_lgb, weights_test)\n\nprint(f\"Train Normalized Weighted RMSLE: {wrmsle_train_lgb}, Test Normalized Weighted RMSLE: {wrmsle_test_lgb}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-14T17:39:26.819929Z","iopub.execute_input":"2025-01-14T17:39:26.820262Z","iopub.status.idle":"2025-01-14T17:46:58.898010Z","shell.execute_reply.started":"2025-01-14T17:39:26.820235Z","shell.execute_reply":"2025-01-14T17:46:58.896248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"LightGBM regressor got normalized weighted rmsle of 0.84 on test while baseline model of DummyRegressor with mean strategy got over 1. I was not able to do proper hyperparameter tuning since there was not enough RAM in kaggle. ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}