{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:44:19.680578Z","iopub.execute_input":"2025-01-17T10:44:19.680984Z","iopub.status.idle":"2025-01-17T10:44:22.091017Z","shell.execute_reply.started":"2025-01-17T10:44:19.680954Z","shell.execute_reply":"2025-01-17T10:44:22.090131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        paths.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:44:22.092323Z","iopub.execute_input":"2025-01-17T10:44:22.092712Z","iopub.status.idle":"2025-01-17T10:44:22.101547Z","shell.execute_reply.started":"2025-01-17T10:44:22.092687Z","shell.execute_reply":"2025-01-17T10:44:22.100622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:44:22.103786Z","iopub.execute_input":"2025-01-17T10:44:22.10407Z","iopub.status.idle":"2025-01-17T10:44:27.953866Z","shell.execute_reply.started":"2025-01-17T10:44:22.104046Z","shell.execute_reply":"2025-01-17T10:44:27.952413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfor filenames in paths:\n    with py7zr.SevenZipFile(filenames, mode='r') as z_ref:\n        z_ref.extractall(path='/kaggle/working')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:44:27.956401Z","iopub.execute_input":"2025-01-17T10:44:27.956893Z","iopub.status.idle":"2025-01-17T10:45:59.43646Z","shell.execute_reply.started":"2025-01-17T10:44:27.956836Z","shell.execute_reply":"2025-01-17T10:45:59.435253Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_data.csv`** – the test dataset used to evaluate the model's performance.","metadata":{}},{"cell_type":"code","source":"%%time\ndata = pd.read_csv(\"/kaggle/working/train.csv\")\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:45:59.437698Z","iopub.execute_input":"2025-01-17T10:45:59.438015Z","iopub.status.idle":"2025-01-17T10:47:40.855086Z","shell.execute_reply.started":"2025-01-17T10:45:59.437986Z","shell.execute_reply":"2025-01-17T10:47:40.85391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:47:40.856222Z","iopub.execute_input":"2025-01-17T10:47:40.856514Z","iopub.status.idle":"2025-01-17T10:47:48.442681Z","shell.execute_reply.started":"2025-01-17T10:47:40.856487Z","shell.execute_reply":"2025-01-17T10:47:48.441346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[data[\"date\"] < THRESHOLD_TRAIN_DATE].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:47:48.444148Z","iopub.execute_input":"2025-01-17T10:47:48.444502Z","iopub.status.idle":"2025-01-17T10:55:13.626497Z","shell.execute_reply.started":"2025-01-17T10:47:48.444472Z","shell.execute_reply":"2025-01-17T10:55:13.624504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\"/kaggle/working/train_data.csv\")\ntest_df = pd.read_csv(\"/kaggle/working/test_data.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:55:13.630981Z","iopub.execute_input":"2025-01-17T10:55:13.63135Z","iopub.status.idle":"2025-01-17T10:56:54.018851Z","shell.execute_reply.started":"2025-01-17T10:55:13.631321Z","shell.execute_reply":"2025-01-17T10:56:54.017513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"date\"] = pd.to_datetime(df[\"date\"])\ntest_df[\"date\"] = pd.to_datetime(test_df[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:56:54.021426Z","iopub.execute_input":"2025-01-17T10:56:54.021942Z","iopub.status.idle":"2025-01-17T10:57:05.840038Z","shell.execute_reply.started":"2025-01-17T10:56:54.021889Z","shell.execute_reply":"2025-01-17T10:57:05.839002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:57:05.841265Z","iopub.execute_input":"2025-01-17T10:57:05.841555Z","iopub.status.idle":"2025-01-17T10:57:12.050628Z","shell.execute_reply.started":"2025-01-17T10:57:05.84153Z","shell.execute_reply":"2025-01-17T10:57:12.049648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:57:12.05179Z","iopub.execute_input":"2025-01-17T10:57:12.052141Z","iopub.status.idle":"2025-01-17T10:57:13.369891Z","shell.execute_reply.started":"2025-01-17T10:57:12.052067Z","shell.execute_reply":"2025-01-17T10:57:13.368667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:57:13.37137Z","iopub.execute_input":"2025-01-17T10:57:13.371671Z","iopub.status.idle":"2025-01-17T10:57:13.390515Z","shell.execute_reply.started":"2025-01-17T10:57:13.371644Z","shell.execute_reply":"2025-01-17T10:57:13.389308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"start_date = pd.to_datetime(\"2016-04-16\")\nend_date = start_date + pd.DateOffset(months=1)\n\n# Remote time that earthqua happen and 1 month after that \ndf = df[~((df['date'] >= start_date) & (df['date'] <= end_date))]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Memory optimization https://www.kaggle.com/code/jagangupta/memory-optimization-and-eda-on-entire-dataset/notebook\nimport pandas as pd\nimport numpy as np\n\ndef optimize(df):\n    df['Year'] = pd.to_datetime(df['date']).dt.year.astype(np.uint16)  \n    df['Month'] = pd.to_datetime(df['date']).dt.month.astype(np.uint8)  \n    df['Day'] = pd.to_datetime(df['date']).dt.day.astype(np.uint8) \n    del df['date']  \n\n    df['id'] = df['id'].astype(np.int32)\n    df['store_nbr'] = df['store_nbr'].astype(np.int8)\n    df['item_nbr'] = df['item_nbr'].astype(np.uint32)\n    df['unit_sales'] = df['unit_sales'].astype(np.float32)\n\n    return df \n\nprint(\"Memory usage before optimization:\")\nprint(df.memory_usage(deep=True))\n\ndf = optimize(df)\ntest_df = optimize(test_df)\n\nprint(\"Memory usage after optimization 1:\")\nprint(df.memory_usage(deep=True))\n\nprint(\"\\nData types after optimization:\")\nprint(df.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:57:13.391527Z","iopub.execute_input":"2025-01-17T10:57:13.39183Z","iopub.status.idle":"2025-01-17T10:58:17.924963Z","shell.execute_reply.started":"2025-01-17T10:57:13.391796Z","shell.execute_reply":"2025-01-17T10:58:17.92375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def optimize2(df):\n    df[\"onpromotion\"].fillna(-1, inplace=True)\n    df[\"onpromotion\"]=df[\"onpromotion\"].astype(np.int8)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:17.926096Z","iopub.execute_input":"2025-01-17T10:58:17.926438Z","iopub.status.idle":"2025-01-17T10:58:17.931494Z","shell.execute_reply.started":"2025-01-17T10:58:17.926407Z","shell.execute_reply":"2025-01-17T10:58:17.930144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimize2(df);\noptimize2(test_df);\n\nprint(\"Memory usage after optimization 1:\")\nprint(df.memory_usage(deep=True))\n\nprint(\"\\nData types after optimization:\")\nprint(df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:17.932754Z","iopub.execute_input":"2025-01-17T10:58:17.933161Z","iopub.status.idle":"2025-01-17T10:58:35.860215Z","shell.execute_reply.started":"2025-01-17T10:58:17.933121Z","shell.execute_reply":"2025-01-17T10:58:35.85893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# years = ['2013', '2014', '2015', '2016', '2017'] \n# train_by_year = {year: df.loc[df['Year'] == int(year)] for year in years}\n\n# let use ds to train from year > 2016 cause of kaggle's limit \n\ndf = df.loc[df['Year'] >= 2016];\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:35.861265Z","iopub.execute_input":"2025-01-17T10:58:35.861643Z","iopub.status.idle":"2025-01-17T10:58:39.169432Z","shell.execute_reply.started":"2025-01-17T10:58:35.861613Z","shell.execute_reply":"2025-01-17T10:58:39.168119Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"execution":{"iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z"}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:39.170502Z","iopub.execute_input":"2025-01-17T10:58:39.170899Z","iopub.status.idle":"2025-01-17T10:58:39.191858Z","shell.execute_reply.started":"2025-01-17T10:58:39.170861Z","shell.execute_reply":"2025-01-17T10:58:39.190886Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Pay special attention to the transferred column. A holiday that is transferred officially falls on that calendar day, but was moved to another date by the government. A transferred day is more like a normal day than a holiday. To find the day that it was actually celebrated, look for the corresponding row where type is Transfer. For example, the holiday Independencia de Guayaquil was transferred from 2012-10-09 to 2012-10-12, which means it was celebrated on 2012-10-12. Days that are type Bridge are extra days that are added to a holiday (e.g., to extend the break across a long weekend). These are frequently made up by the type Work Day which is a day not normally scheduled for work (e.g., Saturday) that is meant to payback the Bridge.","metadata":{}},{"cell_type":"code","source":"\nholiday_events = holiday_events.loc[(holiday_events['transferred'] == False) & (holiday_events['type'] != 'Work Day')]\nholiday_events = holiday_events.drop(columns=['transferred'])\nprint(holiday_events);\n\nholiday_events['type'] = holiday_events['type'].replace({\n    'Holiday': 1,\n    'Event': 2,\n    'Additional': 3,\n    'Transfer': 4,\n    'Bridge': 5\n}).fillna(0).astype(np.int8)  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:39.192979Z","iopub.execute_input":"2025-01-17T10:58:39.193347Z","iopub.status.idle":"2025-01-17T10:58:39.207358Z","shell.execute_reply.started":"2025-01-17T10:58:39.193293Z","shell.execute_reply":"2025-01-17T10:58:39.206422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_events['date'] = pd.to_datetime(holiday_events['date'])\n\nholiday_events['Year'] = pd.to_datetime(holiday_events['date']).dt.year.astype(np.uint16)  \nholiday_events['Month'] = pd.to_datetime(holiday_events['date']).dt.month.astype(np.uint8)  \nholiday_events['Day'] = pd.to_datetime(holiday_events['date']).dt.day.astype(np.uint8) \n\ndel holiday_events['date']  \ndel holiday_events['description']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:39.208419Z","iopub.execute_input":"2025-01-17T10:58:39.20879Z","iopub.status.idle":"2025-01-17T10:58:39.235334Z","shell.execute_reply.started":"2025-01-17T10:58:39.20876Z","shell.execute_reply":"2025-01-17T10:58:39.234014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(\n    holiday_events[['Day', 'Month', 'Year', 'type', 'locale']],\n    on=['Day', 'Month', 'Year'],\n    how='left'\n);\n\ntest_df = test_df.merge(\n    holiday_events[['Day', 'Month', 'Year', 'type', 'locale']],\n    on=['Day', 'Month', 'Year'],\n    how='left'\n);\n\n\ndf['is_holiday'] = df['type'].notna().astype(np.int8);\ntest_df['is_holiday'] = test_df['type'].notna().astype(np.int8);\n\ndel holiday_events;","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:39.236366Z","iopub.execute_input":"2025-01-17T10:58:39.236784Z","iopub.status.idle":"2025-01-17T10:58:48.011991Z","shell.execute_reply.started":"2025-01-17T10:58:39.236742Z","shell.execute_reply":"2025-01-17T10:58:48.010871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def handle_type_holiday(row):\n    if row['type'] == 'Work Day': \n        row['is_holiday'] = 0\n    return row\n\ndf = df.apply(handle_type_holiday, axis=1)\ntest_df = test_df.apply(handle_type_holiday, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-17T10:58:48.013206Z","iopub.execute_input":"2025-01-17T10:58:48.013533Z","execution_failed":"2025-01-17T11:10:58.381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df['unit_sales'].min())\nprint(df['unit_sales'].max())","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Categorize holidays by their type and consider their impact on sales.\n\n","metadata":{}},{"cell_type":"code","source":"# df_copy = df.copy()\n\n# df_copy['holiday_group'] = df_copy.apply(\n#     lambda row: 'Not a Holiday' if row['is_holiday'] == 0 else row['type'], axis=1\n# )\n\n# sales_by_holiday = df_copy.groupby('holiday_group')['unit_sales'].sum().reset_index()\n\n# plt.figure(figsize=(8, 6))\n# sns.barplot(data=sales_by_holiday, x='holiday_group', y='unit_sales', palette='coolwarm')\n\n# plt.title('Total Unit Sales by Holiday Group', fontsize=16)\n# plt.xlabel('Holiday Group', fontsize=14)\n# plt.ylabel('Total Unit Sales', fontsize=14)\n# plt.xticks(fontsize=12, rotation=30, ha='right')\n# plt.yticks(fontsize=12)\n\n# plt.tight_layout()\n# plt.show()\n\n# del df_copy, sales_by_holiday","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.383Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Use the locale column to analyze regional effects.","metadata":{}},{"cell_type":"code","source":"# sales_by_locale = df.groupby('locale')['unit_sales'].sum().reset_index()\n\n# plt.figure(figsize=(8, 6))\n# sns.barplot(data=sales_by_locale, x='locale', y='unit_sales', palette='viridis')\n\n# plt.title('Total Unit Sales by Locale', fontsize=16)\n# plt.xlabel('Locale', fontsize=14)\n# plt.ylabel('Total Unit Sales', fontsize=14)\n# plt.xticks(fontsize=12)\n# plt.yticks(fontsize=12)\n\n# plt.show()\n\n# del sales_by_locale","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.383Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\nprint(items.shape)\nitems.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# grouped = df.groupby(['family', 'class']).agg(\n#     total_sales=('unit_sales', 'sum'),\n#     avg_sales=('unit_sales', 'mean'),\n#     total_items=('unit_sales', 'count')\n# ).reset_index()\n\n# print(grouped)\n\n# plt.figure(figsize=(10, 6))\n# sns.barplot(data=grouped, x='family', y='total_sales', hue='class', palette='viridis')\n\n# plt.title('Total Unit Sales by Family and Class', fontsize=16)\n# plt.xlabel('Family', fontsize=14)\n# plt.ylabel('Total Unit Sales', fontsize=14)\n# plt.xticks(fontsize=12)\n# plt.legend(title='Class', fontsize=12)\n\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(items[['item_nbr', 'family', 'class', 'perishable']], on='item_nbr', how='left')\ntest_df = test_df.merge(items[['item_nbr', 'family', 'class', 'perishable']], on='item_nbr', how='left')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.384Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\nprint(oil.shape)\noil.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil['date'] = pd.to_datetime(oil['date'])\n\noil['Year'] = pd.to_datetime(oil['date']).dt.year.astype(np.uint16)  \noil['Month'] = pd.to_datetime(oil['date']).dt.month.astype(np.uint8)  \noil['Day'] = pd.to_datetime(oil['date']).dt.day.astype(np.uint8) \n\ndel oil['date']  ","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(oil['dcoilwtico'].min(), oil['dcoilwtico'].max());\noil['dcoilwtico'] = oil['dcoilwtico'].astype(np.float32);","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(oil.dtypes);\ndf = df.merge(oil[['dcoilwtico']], on=['Year', 'Month', 'Day'], how='left')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.\n","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()\ndf = df.rename(columns={'type': 'store_type'})","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.merge(stores, on='store_nbr', how='left')\n\ndel stores;","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# they paid salary earch 2 week so let mark day 13, 14, 15 and 29, 30, 31 \nday = [13, 14, 15, 29, 30, 31]\ndf['salary_day'] = df['Day'].apply(lambda x: 1 if x in day else 0).astype(np.int8)  ","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-17T11:10:58.386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"code","source":"\ny_train = df.pop('unit_sales')  \nX_train = df  \ny_test = test_df.pop('unit_sales')  \nX_test = test_df ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.model_selection import GridSearchCV\n\nimport lightgbm as lgb","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric = ['class', 'dcoilwtico', 'perishable', 'store_type']\ncategorical = ['onpromotion', 'family', 'class', 'type', 'cluster', 'is_holiday', 'salary_day']\nlabel = 'unit_sales'\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', StandardScaler(), numeric),\n        ('cat', OneHotEncoder(handle_unknown='ignore'), categorical)\n    ]\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" Items marked as perishable have a score weight of 1.25; otherwise, the weight is 1.0.\n ","metadata":{}},{"cell_type":"code","source":"def normalized_weighted_rmsle(y_true, y_pred, weights):\n\n    log_y_true = np.log1p(y_true)\n    log_y_pred = np.log1p(y_pred)\n\n    log_errors = log_y_pred - log_y_true\n    \n    squared_log_errors = log_errors ** 2\n    \n    weighted_squared_log_errors = weights * squared_log_errors\n    \n    weighted_error_sum = np.sum(weighted_squared_log_errors)\n    total_weights = np.sum(weights)\n    \n    normalized_weighted_mean = weighted_error_sum / total_weights\n    return np.sqrt(normalized_weighted_mean)\n\nweights_train = np.where(X_train['perishable'] == 1, 1.25, 1)\nweights_test = np.where(X_test['perishable'] == 1, 1.25, 1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dummy_reg.fit(X_train, y_train)\n\ny_train_pred_dummy = dummy_reg.predict(X_train)\ny_test_pred_dummy = dummy_reg.predict(X_test)\n\nwrmsle_train_dummy = normalized_weighted_rmsle(y_train, y_train_pred_dummy, weights_train)\nwrmsle_test_dummy = normalized_weighted_rmsle(y_test, y_test_pred_dummy, weights_test)\n\nprint(f\"Train Normalized Weighted RMSLE: {wrmsle_train_dummy}, Test Normalized Weighted RMSLE: {wrmsle_test_dummy}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install catboost\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"catboost_reg = Pipeline(steps=[\n    ('preprocessor', preprocessor),  \n    ('regressor', CatBoostRegressor(iterations=500, learning_rate=0.1, depth=6, cat_features=categorical, verbose=0))  # CatBoost regressor\n])\n\ncatboost_reg.fit(X_train, y_train)\n\ny_train_pred_catboost = catboost_reg.predict(X_train)\ny_test_pred_catboost = catboost_reg.predict(X_test)\n\nwrmsle_train_catboost = normalized_weighted_rmsle(y_train, y_train_pred_catboost, weights_train)\nwrmsle_test_catboost = normalized_weighted_rmsle(y_test, y_test_pred_catboost, weights_test)\n\nprint(f\"Train Normalized Weighted RMSLE: {wrmsle_train_catboost}, Test Normalized Weighted RMSLE: {wrmsle_test_catboost}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}