{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:14:50.175133Z","iopub.execute_input":"2025-01-12T13:14:50.175522Z","iopub.status.idle":"2025-01-12T13:14:51.446626Z","shell.execute_reply.started":"2025-01-12T13:14:50.175493Z","shell.execute_reply":"2025-01-12T13:14:51.445601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        paths.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:14:51.447796Z","iopub.execute_input":"2025-01-12T13:14:51.448179Z","iopub.status.idle":"2025-01-12T13:14:51.456472Z","shell.execute_reply.started":"2025-01-12T13:14:51.448154Z","shell.execute_reply":"2025-01-12T13:14:51.455569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:14:51.458295Z","iopub.execute_input":"2025-01-12T13:14:51.458561Z","iopub.status.idle":"2025-01-12T13:14:59.376708Z","shell.execute_reply.started":"2025-01-12T13:14:51.458536Z","shell.execute_reply":"2025-01-12T13:14:59.375591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfor filenames in paths:\n    with py7zr.SevenZipFile(filenames, mode='r') as z_ref:\n        z_ref.extractall(path='/kaggle/working')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:14:59.378595Z","iopub.execute_input":"2025-01-12T13:14:59.378977Z","iopub.status.idle":"2025-01-12T13:16:28.507607Z","shell.execute_reply.started":"2025-01-12T13:14:59.378940Z","shell.execute_reply":"2025-01-12T13:16:28.506679Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_data.csv`** – the test dataset used to evaluate the model's performance.","metadata":{}},{"cell_type":"code","source":"%%time\ndata = pd.read_csv(\"/kaggle/working/train.csv\")\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:16:28.508538Z","iopub.execute_input":"2025-01-12T13:16:28.508855Z","iopub.status.idle":"2025-01-12T13:18:10.499289Z","shell.execute_reply.started":"2025-01-12T13:16:28.508812Z","shell.execute_reply":"2025-01-12T13:18:10.498319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:18:10.500224Z","iopub.execute_input":"2025-01-12T13:18:10.500482Z","iopub.status.idle":"2025-01-12T13:18:17.903970Z","shell.execute_reply.started":"2025-01-12T13:18:10.500460Z","shell.execute_reply":"2025-01-12T13:18:17.902926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[data[\"date\"] < THRESHOLD_TRAIN_DATE].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:18:17.905044Z","iopub.execute_input":"2025-01-12T13:18:17.905437Z","iopub.status.idle":"2025-01-12T13:25:28.485937Z","shell.execute_reply.started":"2025-01-12T13:18:17.905401Z","shell.execute_reply":"2025-01-12T13:25:28.484725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\"/kaggle/working/train_data.csv\")\ntest_df = pd.read_csv(\"/kaggle/working/test_data.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:25:28.489047Z","iopub.execute_input":"2025-01-12T13:25:28.489365Z","iopub.status.idle":"2025-01-12T13:27:09.118474Z","shell.execute_reply.started":"2025-01-12T13:25:28.489338Z","shell.execute_reply":"2025-01-12T13:27:09.117403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"date\"] = pd.to_datetime(df[\"date\"])\ntest_df[\"date\"] = pd.to_datetime(test_df[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:09.124397Z","iopub.execute_input":"2025-01-12T13:27:09.124721Z","iopub.status.idle":"2025-01-12T13:27:20.912134Z","shell.execute_reply.started":"2025-01-12T13:27:09.124696Z","shell.execute_reply":"2025-01-12T13:27:20.911078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:20.913223Z","iopub.execute_input":"2025-01-12T13:27:20.913549Z","iopub.status.idle":"2025-01-12T13:27:27.134331Z","shell.execute_reply.started":"2025-01-12T13:27:20.913513Z","shell.execute_reply":"2025-01-12T13:27:27.133297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:27.135527Z","iopub.execute_input":"2025-01-12T13:27:27.135944Z","iopub.status.idle":"2025-01-12T13:27:28.394065Z","shell.execute_reply.started":"2025-01-12T13:27:27.135900Z","shell.execute_reply":"2025-01-12T13:27:28.392990Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.395081Z","iopub.execute_input":"2025-01-12T13:27:28.395449Z","iopub.status.idle":"2025-01-12T13:27:28.409849Z","shell.execute_reply.started":"2025-01-12T13:27:28.395415Z","shell.execute_reply":"2025-01-12T13:27:28.408858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df.shape)\nprint(test_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.411014Z","iopub.execute_input":"2025-01-12T13:27:28.411412Z","iopub.status.idle":"2025-01-12T13:27:28.416977Z","shell.execute_reply.started":"2025-01-12T13:27:28.411375Z","shell.execute_reply":"2025-01-12T13:27:28.416052Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"execution":{"iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z"}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.418102Z","iopub.execute_input":"2025-01-12T13:27:28.418485Z","iopub.status.idle":"2025-01-12T13:27:28.449979Z","shell.execute_reply.started":"2025-01-12T13:27:28.418448Z","shell.execute_reply":"2025-01-12T13:27:28.449184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display unique values for the 'type', 'locale', and 'transferred' columns\nprint(\"Unique values in 'type':\", holiday_events['type'].unique())\nprint(\"Unique values in 'locale':\", holiday_events['locale'].unique())\nprint(\"Unique values in 'transferred':\", holiday_events['transferred'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.451010Z","iopub.execute_input":"2025-01-12T13:27:28.451384Z","iopub.status.idle":"2025-01-12T13:27:28.460069Z","shell.execute_reply.started":"2025-01-12T13:27:28.451347Z","shell.execute_reply":"2025-01-12T13:27:28.458761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of the types of the events\ntype_counts = holiday_events['type'].value_counts()\n\nplt.figure(figsize=(10, 6))\ntype_counts.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title('Distribution of Event Types', fontsize=16)\nplt.xlabel('Event Type', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.461400Z","iopub.execute_input":"2025-01-12T13:27:28.461788Z","iopub.status.idle":"2025-01-12T13:27:28.703505Z","shell.execute_reply.started":"2025-01-12T13:27:28.461749Z","shell.execute_reply":"2025-01-12T13:27:28.702419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of the levels of events\nlocale_counts = holiday_events['locale'].value_counts()\n\nplt.figure(figsize=(8, 5))\nlocale_counts.plot(kind='bar', color='orange', edgecolor='black')\nplt.title('Distribution of Locale Types', fontsize=16)\nplt.xlabel('Locale', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=0)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.704588Z","iopub.execute_input":"2025-01-12T13:27:28.704986Z","iopub.status.idle":"2025-01-12T13:27:28.895510Z","shell.execute_reply.started":"2025-01-12T13:27:28.704948Z","shell.execute_reply":"2025-01-12T13:27:28.894501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract the unique years\nholiday_events['date'] = pd.to_datetime(holiday_events['date'])\nunique_years = holiday_events['date'].dt.year.unique()\nprint(\"Unique years in the 'date' column:\", unique_years)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.896618Z","iopub.execute_input":"2025-01-12T13:27:28.896995Z","iopub.status.idle":"2025-01-12T13:27:28.905713Z","shell.execute_reply.started":"2025-01-12T13:27:28.896957Z","shell.execute_reply":"2025-01-12T13:27:28.904693Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### A short remark: the transferred column indicates whether a holiday has been officially moved to another date by the government. Ex. transferred = True means that the holiday does not occur on its usual or official date. Instead, it has been transferred to another date.","metadata":{}},{"cell_type":"code","source":"# Count the number of transferred and non-transferred holidays\ntransferred_counts = holiday_events['transferred'].value_counts()\nprint(\"Holidays not transferred (False):\", transferred_counts.get(False, 0))\nprint(\"Holidays transferred (True):\", transferred_counts.get(True, 0))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.906890Z","iopub.execute_input":"2025-01-12T13:27:28.907303Z","iopub.status.idle":"2025-01-12T13:27:28.926527Z","shell.execute_reply.started":"2025-01-12T13:27:28.907261Z","shell.execute_reply":"2025-01-12T13:27:28.925301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for NaN values in the dataset holiday_events\nnan_values = holiday_events.isna().sum()\nprint(\"Number of NaN values in each column:\")\nprint(nan_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.927740Z","iopub.execute_input":"2025-01-12T13:27:28.928164Z","iopub.status.idle":"2025-01-12T13:27:28.945661Z","shell.execute_reply.started":"2025-01-12T13:27:28.928102Z","shell.execute_reply":"2025-01-12T13:27:28.944585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add a flag indicating whether a date is a holiday to the training and test sets\n# Extract unique holiday dates from holiday_events dataset\nholiday_dates = holiday_events['date'].unique()\n# Add a flag for holidays using .isin()\ndf['is_holiday'] = df['date'].isin(holiday_dates).astype(int)\ntest_df['is_holiday'] = test_df['date'].isin(holiday_dates).astype(int)\n\nprint(\"Training set with holiday flag:\\n\", df.head())\nprint(\"\\nTest set with holiday flag:\\n\", test_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:28.946643Z","iopub.execute_input":"2025-01-12T13:27:28.946903Z","iopub.status.idle":"2025-01-12T13:27:30.142986Z","shell.execute_reply.started":"2025-01-12T13:27:28.946881Z","shell.execute_reply":"2025-01-12T13:27:30.141886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by is_holiday to calculate average sales\nholiday_sales = df.groupby('is_holiday')['unit_sales'].mean()\n\nplt.figure(figsize=(6, 4))\nholiday_sales.plot(kind='bar', color=['skyblue', 'orange'])\nplt.title('Average Sales: Holiday vs Non-Holiday', fontsize=16)\nplt.xlabel('Is Holiday', fontsize=12)\nplt.ylabel('Average Sales', fontsize=12)\nplt.xticks([0, 1], labels=['Non-Holiday', 'Holiday'], rotation=0)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:30.144309Z","iopub.execute_input":"2025-01-12T13:27:30.144700Z","iopub.status.idle":"2025-01-12T13:27:32.671072Z","shell.execute_reply.started":"2025-01-12T13:27:30.144663Z","shell.execute_reply":"2025-01-12T13:27:32.670027Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### We can see that these to classes are balanced. Consuming behavior doesn't differ a lot between holiday-days and non-holiday-days. Anyway, on holidays the demand is hidher than on no-holidays.","metadata":{}},{"cell_type":"code","source":"# Categorize holidays by their type and consider their impact on sales\n# Create a dictionary mapping dates to their event types\nevent_type_map = holiday_events.set_index('date')['type'].to_dict()\n\ndf['event_type'] = df['date'].map(event_type_map).fillna('No Holiday')\ntest_df['event_type'] = test_df['date'].map(event_type_map).fillna('No Holiday')\n\nprint(\"Training set with holiday types:\\n\", df.head())\nprint(\"\\nTest set with holiday types:\\n\", test_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:32.676117Z","iopub.execute_input":"2025-01-12T13:27:32.676471Z","iopub.status.idle":"2025-01-12T13:27:44.451479Z","shell.execute_reply.started":"2025-01-12T13:27:32.676442Z","shell.execute_reply":"2025-01-12T13:27:44.450225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use the locale column to analyze regional effects\n# Create a dictionary mapping dates to their locale\nlocale_map = holiday_events.set_index('date')['locale'].to_dict()\n\ndf['locale_impact'] = df['date'].map(locale_map).fillna('None')\ntest_df['locale_impact'] = test_df['date'].map(locale_map).fillna('None')\n\nprint(\"Training set with locale impact:\\n\", df.head())\nprint(\"\\nTest set with locale impact:\\n\", test_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:44.453962Z","iopub.execute_input":"2025-01-12T13:27:44.454313Z","iopub.status.idle":"2025-01-12T13:27:56.756712Z","shell.execute_reply.started":"2025-01-12T13:27:44.454283Z","shell.execute_reply":"2025-01-12T13:27:56.755747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\nprint(items.shape)\nitems.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:56.757920Z","iopub.execute_input":"2025-01-12T13:27:56.758235Z","iopub.status.idle":"2025-01-12T13:27:56.779681Z","shell.execute_reply.started":"2025-01-12T13:27:56.758201Z","shell.execute_reply":"2025-01-12T13:27:56.778649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group items by 'family' and 'class' and count the number of items in each group\nfamily_class_group = items.groupby(['family', 'class']).size().reset_index(name='item_count')\n\n# Analyze trends by family\nfamily_trends = family_class_group.groupby('family')['item_count'].sum().reset_index()\nfamily_trends = family_trends.sort_values(by='item_count', ascending=False)\n\n# Plot the item count by family\nplt.figure(figsize=(20, 6))\nplt.bar(family_trends['family'], family_trends['item_count'], color='skyblue', edgecolor='black')\nplt.title('Item Count by Family', fontsize=16)\nplt.xlabel('Family', fontsize=12)\nplt.ylabel('Item Count', fontsize=12)\nplt.xticks(rotation=90)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:56.780800Z","iopub.execute_input":"2025-01-12T13:27:56.781209Z","iopub.status.idle":"2025-01-12T13:27:57.252189Z","shell.execute_reply.started":"2025-01-12T13:27:56.781171Z","shell.execute_reply":"2025-01-12T13:27:57.250915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group items by family and class and calculate the count of items\nfamily_class_group = items.groupby(['family', 'class']).size().unstack(fill_value=0)\n\n# Plot a stacked bar chart for family and class distribution\nfamily_class_group.T.plot(kind='bar', stacked=True, figsize=(14, 8), colormap='tab20')\nplt.title('Class Distribution Within Families', fontsize=16)\nplt.xlabel('Class', fontsize=12)\nplt.ylabel('Item Count', fontsize=12)\nplt.legend(title='Family', bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:27:57.253478Z","iopub.execute_input":"2025-01-12T13:27:57.253930Z","iopub.status.idle":"2025-01-12T13:28:13.904460Z","shell.execute_reply.started":"2025-01-12T13:27:57.253885Z","shell.execute_reply":"2025-01-12T13:28:13.903468Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### This plot is not very informative as soon as there are a lot of classes, so let's better represent the heatmap between TOP-12 families and classes","metadata":{}},{"cell_type":"code","source":"# Plot heatmap\nimport seaborn as sns\ntop_families = family_class_group.sum(axis=1).nlargest(12).index\nheatmap_data = family_class_group.loc[top_families]\n\nplt.figure(figsize=(14, 8))\nsns.heatmap(heatmap_data, cmap='YlGnBu', annot=False, fmt='d', cbar=True)\nplt.title('Class Distribution Heatmap for Top 12 Families', fontsize=16)\nplt.xlabel('Class', fontsize=12)\nplt.ylabel('Family', fontsize=12)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:13.905532Z","iopub.execute_input":"2025-01-12T13:28:13.905789Z","iopub.status.idle":"2025-01-12T13:28:14.623867Z","shell.execute_reply.started":"2025-01-12T13:28:13.905768Z","shell.execute_reply":"2025-01-12T13:28:14.622777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Map the weights of the perishable column to the training and test datasets using the item_nbr column\nweights_map = items.set_index('item_nbr')['perishable'].apply(lambda x: 1.25 if x == 1 else 1.0).to_dict()\n\ndf['weight'] = df['item_nbr'].map(weights_map)\ntest_df['weight'] = test_df['item_nbr'].map(weights_map)\n\nprint(\"Training set weights:\\n\", df[['item_nbr', 'weight']].head())\nprint(\"\\nTest set weights:\\n\", test_df[['item_nbr', 'weight']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:14.624919Z","iopub.execute_input":"2025-01-12T13:28:14.625249Z","iopub.status.idle":"2025-01-12T13:28:17.600402Z","shell.execute_reply.started":"2025-01-12T13:28:14.625210Z","shell.execute_reply":"2025-01-12T13:28:17.599420Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\nprint(oil.shape)\noil.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:17.601277Z","iopub.execute_input":"2025-01-12T13:28:17.601532Z","iopub.status.idle":"2025-01-12T13:28:17.620718Z","shell.execute_reply.started":"2025-01-12T13:28:17.601511Z","shell.execute_reply":"2025-01-12T13:28:17.619413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:17.621791Z","iopub.execute_input":"2025-01-12T13:28:17.622199Z","iopub.status.idle":"2025-01-12T13:28:17.635604Z","shell.execute_reply.started":"2025-01-12T13:28:17.622160Z","shell.execute_reply":"2025-01-12T13:28:17.634491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Missing values in dcoilwtico:\", oil['dcoilwtico'].isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:17.636803Z","iopub.execute_input":"2025-01-12T13:28:17.637174Z","iopub.status.idle":"2025-01-12T13:28:17.655833Z","shell.execute_reply.started":"2025-01-12T13:28:17.637128Z","shell.execute_reply":"2025-01-12T13:28:17.654653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil['date'] = pd.to_datetime(oil['date'])\n# Fill missing values in dcoilwtico using forward and backward fill\noil['dcoilwtico'] = oil['dcoilwtico'].ffill().bfill()\nprint(\"Missing values in dcoilwtico after filling:\", oil['dcoilwtico'].isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:17.656836Z","iopub.execute_input":"2025-01-12T13:28:17.657233Z","iopub.status.idle":"2025-01-12T13:28:17.676047Z","shell.execute_reply.started":"2025-01-12T13:28:17.657192Z","shell.execute_reply":"2025-01-12T13:28:17.674998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil_price_map = oil.set_index('date')['dcoilwtico'].to_dict() # Create a dictionary mapping date to dcoilwtico\n\n# Map oil prices to the training and test datasets based on the date\ndf['oil_price'] = df['date'].map(oil_price_map)\ntest_df['oil_price'] = test_df['date'].map(oil_price_map)\n\n# Check for missing oil prices in training and test datasets\nmissing_train = df['oil_price'].isna().sum()\nmissing_test = test_df['oil_price'].isna().sum()\n\nprint(\"Missing oil prices in training set:\", missing_train)\nprint(\"Missing oil prices in test set:\", missing_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:17.677184Z","iopub.execute_input":"2025-01-12T13:28:17.677549Z","iopub.status.idle":"2025-01-12T13:28:20.046224Z","shell.execute_reply.started":"2025-01-12T13:28:17.677512Z","shell.execute_reply":"2025-01-12T13:28:20.045249Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### It means that there are some dates that exist in oil dataset and don't exist in the training and test data. So, I need to fix it for further EDA.","metadata":{}},{"cell_type":"code","source":"# Find dates in df that are missing in the oil dataset\nmissing_dates_train = df[~df['date'].isin(oil['date'])]['date'].unique()\nprint(\"Missing dates in training set:\", missing_dates_train)\n\n# Find dates in test_df that are missing in the oil dataset\nmissing_dates_test = test_df[~test_df['date'].isin(oil['date'])]['date'].unique()\nprint(\"Missing dates in test set:\", missing_dates_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:20.047088Z","iopub.execute_input":"2025-01-12T13:28:20.047448Z","iopub.status.idle":"2025-01-12T13:28:24.967947Z","shell.execute_reply.started":"2025-01-12T13:28:20.047421Z","shell.execute_reply":"2025-01-12T13:28:24.967007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_dates = pd.date_range(start=min(df['date'].min(), test_df['date'].min()), \n                          end=max(df['date'].max(), test_df['date'].max())) # Create a full date range from the training and test sets\n\noil = oil.set_index('date').reindex(all_dates).reset_index() # Reindex the oil dataset to include all dates, and fill missing values\noil.rename(columns={'index': 'date'}, inplace=True)\n\noil['dcoilwtico'] = oil['dcoilwtico'].ffill().bfill() # Fill missing oil prices","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:24.968991Z","iopub.execute_input":"2025-01-12T13:28:24.969310Z","iopub.status.idle":"2025-01-12T13:28:25.555291Z","shell.execute_reply.started":"2025-01-12T13:28:24.969283Z","shell.execute_reply":"2025-01-12T13:28:25.554278Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Now, we are ready for filling in the missing values in the training and test sets as soon as the dates exist in oil dataset set. ","metadata":{}},{"cell_type":"code","source":"oil_price_map = oil.set_index('date')['dcoilwtico'].to_dict() # Create a dictionary mapping date to dcoilwtico\n\n# Map oil prices to the training and test datasets based on the date\ndf['oil_price'] = df['date'].map(oil_price_map)\ntest_df['oil_price'] = test_df['date'].map(oil_price_map)\n\n# Check for missing oil prices in training and test datasets\nmissing_train = df['oil_price'].isna().sum()\nmissing_test = test_df['oil_price'].isna().sum()\n\nprint(\"Missing oil prices in training set:\", missing_train)\nprint(\"Missing oil prices in test set:\", missing_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:25.556484Z","iopub.execute_input":"2025-01-12T13:28:25.556843Z","iopub.status.idle":"2025-01-12T13:28:27.748746Z","shell.execute_reply.started":"2025-01-12T13:28:25.556769Z","shell.execute_reply":"2025-01-12T13:28:27.747701Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.\n","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:27.749768Z","iopub.execute_input":"2025-01-12T13:28:27.750125Z","iopub.status.idle":"2025-01-12T13:28:27.764561Z","shell.execute_reply.started":"2025-01-12T13:28:27.750097Z","shell.execute_reply":"2025-01-12T13:28:27.763346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(stores.info())\nprint(\"Unique cities:\", stores['city'].unique())\nprint(\"Unique states:\", stores['state'].unique())\nprint(\"Unique store types:\", stores['type'].unique())\nprint(\"Unique clusters:\", stores['cluster'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:27.765609Z","iopub.execute_input":"2025-01-12T13:28:27.765951Z","iopub.status.idle":"2025-01-12T13:28:27.781666Z","shell.execute_reply.started":"2025-01-12T13:28:27.765924Z","shell.execute_reply":"2025-01-12T13:28:27.780435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create mapping dictionaries for each relevant column\ncity_map = stores.set_index('store_nbr')['city'].to_dict()\nstate_map = stores.set_index('store_nbr')['state'].to_dict()\ntype_map = stores.set_index('store_nbr')['type'].to_dict()\ncluster_map = stores.set_index('store_nbr')['cluster'].to_dict()\n\n# Add the mapped columns to the training dataset\ndf['city'] = df['store_nbr'].map(city_map)\ndf['state'] = df['store_nbr'].map(state_map)\ndf['store_type'] = df['store_nbr'].map(type_map)\ndf['cluster'] = df['store_nbr'].map(cluster_map)\n\n# Add the mapped columns to the test dataset\ntest_df['city'] = test_df['store_nbr'].map(city_map)\ntest_df['state'] = test_df['store_nbr'].map(state_map)\ntest_df['store_type'] = test_df['store_nbr'].map(type_map)\ntest_df['cluster'] = test_df['store_nbr'].map(cluster_map)\n\nprint(\"Training set with store information:\\n\", df.head())\nprint(\"\\nTest set with store information:\\n\", test_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:27.782787Z","iopub.execute_input":"2025-01-12T13:28:27.783088Z","iopub.status.idle":"2025-01-12T13:28:46.333513Z","shell.execute_reply.started":"2025-01-12T13:28:27.783060Z","shell.execute_reply":"2025-01-12T13:28:46.332335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by city\ncity_sales = df.groupby('city')['unit_sales'].sum().sort_values(ascending=False)\n\nplt.figure(figsize=(14, 6))\ncity_sales.plot(kind='bar', color='skyblue')\nplt.title('Sales by City', fontsize=16)\nplt.xlabel('City', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:46.334813Z","iopub.execute_input":"2025-01-12T13:28:46.335133Z","iopub.status.idle":"2025-01-12T13:28:58.293181Z","shell.execute_reply.started":"2025-01-12T13:28:46.335100Z","shell.execute_reply":"2025-01-12T13:28:58.292012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by state\nstate_sales = df.groupby('state')['unit_sales'].sum().sort_values(ascending=False)\n\nplt.figure(figsize=(10, 6))\nstate_sales.plot(kind='bar', color='orange')\nplt.title('Sales by State', fontsize=16)\nplt.xlabel('State', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:28:58.294359Z","iopub.execute_input":"2025-01-12T13:28:58.294719Z","iopub.status.idle":"2025-01-12T13:29:08.591866Z","shell.execute_reply.started":"2025-01-12T13:28:58.294689Z","shell.execute_reply":"2025-01-12T13:29:08.590753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by store type\ntype_sales = df.groupby('store_type')['unit_sales'].sum().sort_values(ascending=False)\n\nplt.figure(figsize=(8, 5))\ntype_sales.plot(kind='bar', color='green')\nplt.title('Sales by Store Type', fontsize=16)\nplt.xlabel('Store Type', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:29:08.592937Z","iopub.execute_input":"2025-01-12T13:29:08.593236Z","iopub.status.idle":"2025-01-12T13:29:18.084836Z","shell.execute_reply.started":"2025-01-12T13:29:08.593210Z","shell.execute_reply":"2025-01-12T13:29:18.083598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by cluster\ncluster_sales = df.groupby('cluster')['unit_sales'].sum().sort_values(ascending=False)\n\nplt.figure(figsize=(10, 6))\ncluster_sales.plot(kind='bar', color='purple')\nplt.title('Sales by Store Cluster', fontsize=16)\nplt.xlabel('Cluster', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:29:18.085973Z","iopub.execute_input":"2025-01-12T13:29:18.086387Z","iopub.status.idle":"2025-01-12T13:29:21.072360Z","shell.execute_reply.started":"2025-01-12T13:29:18.086347Z","shell.execute_reply":"2025-01-12T13:29:21.071188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Label encode 'city', 'state', 'store_type', and 'cluster'\nfrom sklearn.preprocessing import LabelEncoder\n\nle_city = LabelEncoder()\nle_state = LabelEncoder()\nle_type = LabelEncoder()\nle_cluster = LabelEncoder()\n\ndf['city_encoded'] = le_city.fit_transform(df['city'])\ndf['state_encoded'] = le_state.fit_transform(df['state'])\ndf['store_type_encoded'] = le_type.fit_transform(df['store_type'])\ndf['cluster_encoded'] = le_cluster.fit_transform(df['cluster'])\n\ntest_df['city_encoded'] = le_city.transform(test_df['city'])\ntest_df['state_encoded'] = le_state.transform(test_df['state'])\ntest_df['store_type_encoded'] = le_type.transform(test_df['store_type'])\ntest_df['cluster_encoded'] = le_cluster.transform(test_df['cluster'])\n\nprint(\"Encoded training set:\\n\", df[['city', 'city_encoded']].head())\nprint(\"\\nEncoded test set:\\n\", test_df[['city', 'city_encoded']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:29:21.073449Z","iopub.execute_input":"2025-01-12T13:29:21.073791Z","iopub.status.idle":"2025-01-12T13:30:31.035232Z","shell.execute_reply.started":"2025-01-12T13:29:21.073760Z","shell.execute_reply":"2025-01-12T13:30:31.034153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add regional sales as a new feature for EDA\nregional_sales_map = df.groupby('state')['unit_sales'].sum().to_dict()\ndf['regional_sales'] = df['state'].map(regional_sales_map)\ntest_df['regional_sales'] = test_df['state'].map(regional_sales_map)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:30:31.036303Z","iopub.execute_input":"2025-01-12T13:30:31.036570Z","iopub.status.idle":"2025-01-12T13:30:48.197311Z","shell.execute_reply.started":"2025-01-12T13:30:31.036540Z","shell.execute_reply":"2025-01-12T13:30:48.196399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by store type and cluster to calculate average sales per store type and per cluster as new features for EDA\navg_sales_per_type = df.groupby('store_type')['unit_sales'].mean()\navg_sales_per_cluster = df.groupby('cluster')['unit_sales'].mean()\n\ntype_avg_sales_map = avg_sales_per_type.to_dict()\ncluster_avg_sales_map = avg_sales_per_cluster.to_dict()\n\ndf['avg_sales_per_type'] = df['store_type'].map(type_avg_sales_map)\ndf['avg_sales_per_cluster'] = df['cluster'].map(cluster_avg_sales_map)\n\ntest_df['avg_sales_per_type'] = test_df['store_type'].map(type_avg_sales_map)\ntest_df['avg_sales_per_cluster'] = test_df['cluster'].map(cluster_avg_sales_map)\n\nprint(\"Training set with new features:\\n\", df[['store_type', 'avg_sales_per_type']].head())\nprint(\"\\nTest set with new features:\\n\", test_df[['store_type', 'avg_sales_per_type']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:30:48.198219Z","iopub.execute_input":"2025-01-12T13:30:48.198555Z","iopub.status.idle":"2025-01-12T13:31:11.977347Z","shell.execute_reply.started":"2025-01-12T13:30:48.198522Z","shell.execute_reply":"2025-01-12T13:31:11.976087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:11.978596Z","iopub.execute_input":"2025-01-12T13:31:11.979064Z","iopub.status.idle":"2025-01-12T13:31:12.005281Z","shell.execute_reply.started":"2025-01-12T13:31:11.979024Z","shell.execute_reply":"2025-01-12T13:31:12.004242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:12.006450Z","iopub.execute_input":"2025-01-12T13:31:12.006902Z","iopub.status.idle":"2025-01-12T13:31:12.031589Z","shell.execute_reply.started":"2025-01-12T13:31:12.006858Z","shell.execute_reply":"2025-01-12T13:31:12.030315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:12.033076Z","iopub.execute_input":"2025-01-12T13:31:12.033544Z","iopub.status.idle":"2025-01-12T13:31:12.046309Z","shell.execute_reply.started":"2025-01-12T13:31:12.033508Z","shell.execute_reply":"2025-01-12T13:31:12.045197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Unique onpromotion:\", df['onpromotion'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:12.047290Z","iopub.execute_input":"2025-01-12T13:31:12.047614Z","iopub.status.idle":"2025-01-12T13:31:16.872158Z","shell.execute_reply.started":"2025-01-12T13:31:12.047585Z","shell.execute_reply":"2025-01-12T13:31:16.870837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count the values for False, True, and NaN\nonpromotion_counts = df['onpromotion'].value_counts(dropna=False)\n\n# Display the counts\nprint(\"Counts of onpromotion categories in the training set:\")\nprint(onpromotion_counts)\n\n# If working with test_df as well\ntest_onpromotion_counts = test_df['onpromotion'].value_counts(dropna=False)\nprint(\"\\nCounts of onpromotion categories in the test set:\")\nprint(test_onpromotion_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:16.873209Z","iopub.execute_input":"2025-01-12T13:31:16.873505Z","iopub.status.idle":"2025-01-12T13:31:20.971561Z","shell.execute_reply.started":"2025-01-12T13:31:16.873478Z","shell.execute_reply":"2025-01-12T13:31:20.970606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### I will consider NaN values in the onpromotion feature as False. And I need to apply it only on the training set. The test set is ready for binary transformation, as there is no NaN values.","metadata":{}},{"cell_type":"code","source":"df['onpromotion'] = df['onpromotion'].fillna(False).astype(int) \ntest_df['onpromotion'] = test_df['onpromotion'].astype(int)\n\nprint(\"Transformed 'onpromotion' column in training set:\\n\", df['onpromotion'].value_counts())\nprint(\"\\nTransformed 'onpromotion' column in test set:\\n\", test_df['onpromotion'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:20.972612Z","iopub.execute_input":"2025-01-12T13:31:20.972942Z","iopub.status.idle":"2025-01-12T13:31:34.817907Z","shell.execute_reply.started":"2025-01-12T13:31:20.972902Z","shell.execute_reply":"2025-01-12T13:31:34.817030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by onpromotion status\npromotion_sales = df.groupby('onpromotion')['unit_sales'].mean()\n\nplt.figure(figsize=(6, 4))\npromotion_sales.plot(kind='bar', color=['skyblue', 'orange'][:len(promotion_sales)])\nplt.title('Average Sales by Promotion Status', fontsize=16)\nplt.xlabel('Promotion Status', fontsize=12)\nplt.ylabel('Average Sales', fontsize=12)\nplt.xticks([0, 1][:len(promotion_sales)], labels=['Not on Promotion', 'On Promotion'], rotation=0)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:34.818899Z","iopub.execute_input":"2025-01-12T13:31:34.819236Z","iopub.status.idle":"2025-01-12T13:31:38.884583Z","shell.execute_reply.started":"2025-01-12T13:31:34.819210Z","shell.execute_reply":"2025-01-12T13:31:38.883595Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### So, we can see that onpromotion feature needs to be taken into account when building a model.","metadata":{}},{"cell_type":"code","source":"# Label encode 'event_type' in the training and test sets\nle_event = LabelEncoder()\ndf['event_type_encoded'] = le_event.fit_transform(df['event_type'].fillna('None'))\ntest_df['event_type_encoded'] = le_event.transform(test_df['event_type'].fillna('None'))\n\nprint(\"Transformed 'event_type' column in training set:\\n\", df[['event_type', 'event_type_encoded']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:31:38.885619Z","iopub.execute_input":"2025-01-12T13:31:38.885937Z","iopub.status.idle":"2025-01-12T13:32:09.415245Z","shell.execute_reply.started":"2025-01-12T13:31:38.885909Z","shell.execute_reply":"2025-01-12T13:32:09.414094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Look at the values of 'locale_impact' column in the training set\nlocale_impact_counts = df['locale_impact'].value_counts(dropna=False)\nprint(locale_impact_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:32:09.416365Z","iopub.execute_input":"2025-01-12T13:32:09.416706Z","iopub.status.idle":"2025-01-12T13:32:11.091581Z","shell.execute_reply.started":"2025-01-12T13:32:09.416675Z","shell.execute_reply":"2025-01-12T13:32:11.090503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Here None means that there was no event on that date. So we need to consider this None value as it is.","metadata":{}},{"cell_type":"code","source":"# Label encode 'locale_impact' in the training and test sets\nle_locale = LabelEncoder()\ndf['locale_impact_encoded'] = le_locale.fit_transform(df['locale_impact'].fillna('None'))\ntest_df['locale_impact_encoded'] = le_locale.transform(test_df['locale_impact'].fillna('None'))\n\nprint(\"Transformed 'locale_impact' column in training set:\\n\", df[['locale_impact', 'locale_impact_encoded']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:32:11.092741Z","iopub.execute_input":"2025-01-12T13:32:11.093092Z","iopub.status.idle":"2025-01-12T13:32:42.046450Z","shell.execute_reply.started":"2025-01-12T13:32:11.093044Z","shell.execute_reply":"2025-01-12T13:32:42.045323Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Now my data with new features and preprocessed features is ready for EDA and Model Training","metadata":{}},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"markdown","source":"### As soon as I didn't manage to solve the memory allocation problem, which I appeared when I was trying to work with the whole training data (I wanted to filter out the otliers in unit_sales, but there was a memory problem, then I was trying to set X_train, X_test, y_train and y_test by taking the whole training and test data, but again failed), so I decided to take just samples from the training and test data to build the models.","metadata":{}},{"cell_type":"code","source":"df_sample = df.sample(frac=0.05, random_state=42) # Sample 5% of the training data\ntest_df_sample = test_df.sample(frac=0.05, random_state=42) # Sample 5% of the test data\n\nprint(\"Original training data shape:\", df.shape)\nprint(\"Sampled training data shape:\", df_sample.shape)\nprint(\"Original test data shape:\", test_df.shape)\nprint(\"Sampled test data shape:\", test_df_sample.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:32:42.048485Z","iopub.execute_input":"2025-01-12T13:32:42.048841Z","iopub.status.idle":"2025-01-12T13:33:01.480399Z","shell.execute_reply.started":"2025-01-12T13:32:42.048791Z","shell.execute_reply":"2025-01-12T13:33:01.479246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select features for the baseline `DummyModel`\nbaseline_features = [\n    'onpromotion',           # Direct impact on sales\n    'is_holiday',            # Holidays often affect sales patterns\n    'oil_price',             # Economic indicator\n    'weight',                # Weight factor for perishable items\n    'regional_sales',        # Aggregate sales data for regional trends\n    'avg_sales_per_type',    # Store type-based sales averages\n    'avg_sales_per_cluster'  # Cluster-level sales averages\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:33:01.481452Z","iopub.execute_input":"2025-01-12T13:33:01.481802Z","iopub.status.idle":"2025-01-12T13:33:01.486605Z","shell.execute_reply.started":"2025-01-12T13:33:01.481770Z","shell.execute_reply":"2025-01-12T13:33:01.485379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare X_train and X_test\nX_train = df_sample[baseline_features]\nX_test = test_df_sample[baseline_features]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:33:01.487772Z","iopub.execute_input":"2025-01-12T13:33:01.488266Z","iopub.status.idle":"2025-01-12T13:33:01.701409Z","shell.execute_reply.started":"2025-01-12T13:33:01.488225Z","shell.execute_reply":"2025-01-12T13:33:01.700369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare y_train and y_test\ny_train = df_sample['unit_sales']\ny_test = test_df_sample['unit_sales']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:33:01.702558Z","iopub.execute_input":"2025-01-12T13:33:01.702973Z","iopub.status.idle":"2025-01-12T13:33:01.707581Z","shell.execute_reply.started":"2025-01-12T13:33:01.702935Z","shell.execute_reply":"2025-01-12T13:33:01.706540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Implement DummyModel\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.metrics import mean_squared_error\n\ndummy_model = DummyRegressor(strategy='mean')\n\ndummy_model.fit(X_train, y_train)\n\ny_pred_dummy = dummy_model.predict(X_test)\n\nbaseline_rmse = mean_squared_error(y_test, y_pred_dummy, squared=False)\nprint(\"Baseline Model RMSE:\", baseline_rmse)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:33:01.708610Z","iopub.execute_input":"2025-01-12T13:33:01.708969Z","iopub.status.idle":"2025-01-12T13:33:01.844095Z","shell.execute_reply.started":"2025-01-12T13:33:01.708932Z","shell.execute_reply":"2025-01-12T13:33:01.842697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of unit_sales\nplt.hist(y_train, bins=50, alpha=0.7, label=\"Training Set\")\nplt.hist(y_test, bins=50, alpha=0.7, label=\"Test Set\")\nplt.title(\"Distribution of Unit Sales\")\nplt.xlabel(\"Unit Sales\")\nplt.ylabel(\"Frequency\")\nplt.legend()\nplt.show()\n\nprint(\"Mean of unit_sales:\", y_train.mean())\nprint(\"Standard deviation of unit_sales:\", y_train.std())\nprint(\"Median of unit_sales:\", y_train.median())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:33:01.845358Z","iopub.execute_input":"2025-01-12T13:33:01.845657Z","iopub.status.idle":"2025-01-12T13:33:02.523041Z","shell.execute_reply.started":"2025-01-12T13:33:01.845621Z","shell.execute_reply":"2025-01-12T13:33:02.521614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### So, if we look at the Mean, Standard Deviation and Median we can make a conclusion that the baseline RMSE = 22.779 is not good result.\n### Mean (8.577): Th baseline RMSE is significantly higher than the mean, indicating that the DummyRegressor's predictions are far from the typical sales value on average.\n### Standard Deviation (22.214): The baseline RMSE is slightly above the standard deviation, which suggests that the baseline model struggles to capture the variability in the data.\n### Median (4.0): The median is much lower than the mean, indicating that most sales values are small, but there are outliers or higher-value sales skewing the distribution.\n### An RMSE close to the standard deviation suggests the baseline model might not handle these high-value sales well.","metadata":{},"attachments":{"87a35778-e8a6-4225-a8ae-7d5d70ba6a91.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAARSURBVBhXY/z4+ct/BgYGBgAY8gPZTr+2PAAAAABJRU5ErkJggg=="}}},{"cell_type":"code","source":"# Select features for LGBMRegressor and CatBoost models\nfeatures = [\n    'onpromotion', 'is_holiday', 'oil_price', 'weight',\n    'city_encoded', 'state_encoded', 'store_type_encoded', 'cluster_encoded',\n    'event_type_encoded', 'locale_impact_encoded',\n    'regional_sales', 'avg_sales_per_type', 'avg_sales_per_cluster'\n]\n\n\nX_train = df_sample[features]\ny_train = df_sample['unit_sales']  \nX_test = test_df_sample[features]\ny_test = test_df_sample['unit_sales']  \n\nweights = test_df_sample['weight'] # Extract weights for evaluation","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:39:39.398272Z","iopub.execute_input":"2025-01-12T13:39:39.398708Z","iopub.status.idle":"2025-01-12T13:39:40.819984Z","shell.execute_reply.started":"2025-01-12T13:39:39.398674Z","shell.execute_reply":"2025-01-12T13:39:40.818794Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Here’s the formula for Normalized Weighted Root Mean Squared Logarithmic Error (NWRMSLE):\n$$\n\\text{NWRMSLE} = \\sqrt{\\frac{\\sum_{i=1}^n w_i \\left( \\ln(\\hat{y}_i + 1) - \\ln(y_i + 1) \\right)^2}{\\sum_{i=1}^n w_i}}\n$$","metadata":{}},{"cell_type":"code","source":"# Initialize the LightGBM Regressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_error\n\nlgbm = LGBMRegressor(\n    random_state=42,\n    objective='regression',\n    n_estimators=500,\n    learning_rate=0.1,\n    max_depth=10,\n    num_leaves=31\n)\n\nlgbm.fit(X_train, y_train)\ny_pred_lgbm = lgbm.predict(X_test)\n\nrmse_lgbm = mean_squared_error(y_test, y_pred_lgbm, squared=False)\nprint(\"LightGBM RMSE:\", rmse_lgbm)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:43:50.470124Z","iopub.execute_input":"2025-01-12T13:43:50.470533Z","iopub.status.idle":"2025-01-12T13:45:41.491370Z","shell.execute_reply.started":"2025-01-12T13:43:50.470505Z","shell.execute_reply":"2025-01-12T13:45:41.490314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def nwrmsle(y_true, y_pred, weights):\n    log_diff = np.log1p(y_pred) - np.log1p(y_true)\n    weighted_log_diff = (weights * (log_diff ** 2)).sum()\n    normalization = weights.sum()\n    return np.sqrt(weighted_log_diff / normalization)\n\nnwrmsle_lgbm = nwrmsle(y_test, y_pred_lgbm, weights)\nprint(\"LightGBM NWRMSLE:\", nwrmsle_lgbm)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparam_grid = {\n    'num_leaves': [31, 50],\n    'max_depth': [-1, 10],\n    'learning_rate': [0.01, 0.1],\n    'n_estimators': [100, 500]\n}\n\ngrid_search = GridSearchCV(estimator=lgbm, param_grid=param_grid, cv=3, scoring='neg_mean_squared_error', verbose=1)\ngrid_search.fit(X_train, y_train)\nprint(\"Best parameters for LightGBM:\", grid_search.best_params_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T13:46:29.979050Z","iopub.execute_input":"2025-01-12T13:46:29.979892Z","execution_failed":"2025-01-12T13:49:03.068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostRegressor\n\ncatboost = CatBoostRegressor(loss_function='RMSE', logging_level='Silent', random_state=42)\n\nparam_grid = {\n    'depth': [6, 10, 15],\n    'learning_rate': [0.01, 0.1, 0.2],\n    'iterations': [500, 1000, 1500],\n    'l2_leaf_reg': [1, 3, 5]\n}\n\ngrid_search_cb = GridSearchCV(estimator=catboost, param_grid=param_grid, cv=3, scoring='neg_mean_squared_error', verbose=1)\ngrid_search_cb.fit(X_train, y_train)\n\nbest_catboost = grid_search_cb.best_estimator_\n\ny_pred_catboost = best_catboost.predict(X_test)\ny_pred_catboost = np.expm1(y_pred_catboost)  # Reverse log-transform\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### The same memory error appeared when I was running the Gridsearch for LGBM Model. (((","metadata":{}}]}