{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"For this task I used my google collab pro because kaggle notebook CPU doesn't want to process data","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:49:32.62611Z","iopub.execute_input":"2025-01-15T10:49:32.626435Z","iopub.status.idle":"2025-01-15T10:49:32.6301Z","shell.execute_reply.started":"2025-01-15T10:49:32.626409Z","shell.execute_reply":"2025-01-15T10:49:32.629328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# List all files in the input directory\ninput_dir = '/kaggle/input'\npaths = []\n\nfor dirname, _, filenames in os.walk(input_dir):\n    for filename in filenames:\n        if filename.endswith('.7z'):  # Only target .7z files\n            file_path = os.path.join(dirname, filename)\n            print(f\"Found file: {file_path}\")\n            paths.append(file_path)\n\nprint(f\"Total .7z files found: {len(paths)}\")\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:52:54.723843Z","iopub.execute_input":"2025-01-15T10:52:54.724186Z","iopub.status.idle":"2025-01-15T10:52:54.733226Z","shell.execute_reply.started":"2025-01-15T10:52:54.724159Z","shell.execute_reply":"2025-01-15T10:52:54.732402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:27:38.471125Z","iopub.execute_input":"2025-01-15T10:27:38.471411Z","iopub.status.idle":"2025-01-15T10:27:41.561915Z","shell.execute_reply.started":"2025-01-15T10:27:38.471387Z","shell.execute_reply":"2025-01-15T10:27:41.560946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\n\n# Extract .7z files\nfor file_path in paths:\n    try:\n        print(f\"Extracting: {file_path}\")\n        with py7zr.SevenZipFile(file_path, mode='r') as z_ref:\n            z_ref.extractall(path='/kaggle/working')  # Extract to working directory\n        print(f\"Extraction complete: {file_path}\")\n    except Exception as e:\n        print(f\"Error extracting {file_path}: {e}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:53:09.319786Z","iopub.execute_input":"2025-01-15T10:53:09.320071Z","iopub.status.idle":"2025-01-15T10:54:17.325418Z","shell.execute_reply.started":"2025-01-15T10:53:09.320047Z","shell.execute_reply":"2025-01-15T10:54:17.324517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_data.csv`** – the test dataset used to evaluate the model's performance.","metadata":{}},{"cell_type":"code","source":"%%time\ndata = pd.read_csv(\"/kaggle/working/train.csv\")\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:54:25.961592Z","iopub.execute_input":"2025-01-15T10:54:25.961893Z","iopub.status.idle":"2025-01-15T10:55:49.923547Z","shell.execute_reply.started":"2025-01-15T10:54:25.961867Z","shell.execute_reply":"2025-01-15T10:55:49.922667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:56:04.985823Z","iopub.execute_input":"2025-01-15T10:56:04.986095Z","iopub.status.idle":"2025-01-15T10:56:11.299824Z","shell.execute_reply.started":"2025-01-15T10:56:04.986076Z","shell.execute_reply":"2025-01-15T10:56:11.298923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[data[\"date\"] < THRESHOLD_TRAIN_DATE].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T10:56:19.030995Z","iopub.execute_input":"2025-01-15T10:56:19.031311Z","iopub.status.idle":"2025-01-15T11:02:07.419094Z","shell.execute_reply.started":"2025-01-15T10:56:19.031281Z","shell.execute_reply":"2025-01-15T11:02:07.418222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\"/kaggle/working/train_data.csv\")\ntest_df = pd.read_csv(\"/kaggle/working/test_data.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"date\"] = pd.to_datetime(df[\"date\"])\ntest_df[\"date\"] = pd.to_datetime(test_df[\"date\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"execution":{"iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z"}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\n#Converrt\nholiday_events[\"date\"] = pd.to_datetime(holiday_events[\"date\"])\n# Merge \ndf = pd.merge(df, holiday_events[['date', 'type', 'locale']], on='date', how='left')\ntest_df = pd.merge(test_df, holiday_events[['date', 'type', 'locale']], on='date', how='left')\n\n# Display \ndf.head(), test_df.head()\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['is_holiday'] = df['type'].notna().astype(int)\ntest_df['is_holiday'] = test_df['type'].notna().astype(int)\ndf['holiday_type'] = df['type']\ntest_df['holiday_type'] = test_df['type']\n\n# Verify changes\ndf.head(), test_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\ndf = pd.merge(df, items, on='item_nbr', how='left')\ntest_df = pd.merge(test_df, items, on='item_nbr', how='left')\nprint(items.shape)\nitems.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count the number of items in each family\nfamily_counts = items['family'].value_counts()\n\n# Plot the distribution of items by family\nplt.figure(figsize=(12, 6))\nfamily_counts.plot(kind='bar', color='skyblue')\nplt.title('Distribution of Items by Family', fontsize=14)\nplt.xlabel('Family', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales data by family\nsales_by_family = df.groupby('family')['unit_sales'].sum().sort_values(ascending=False)\n\n# Plot the sales by family\nplt.figure(figsize=(12, 6))\nsales_by_family.plot(kind='bar', color='lightgreen')\nplt.title('Total Sales by Item Family', fontsize=14)\nplt.xlabel('Family', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add a column to flag perishable items\ndf['is_perishable'] = df['perishable']\ntest_df['is_perishable'] = test_df['perishable']\n\n# Group sales data by perishability\nsales_by_perishability = df.groupby('is_perishable')['unit_sales'].sum()\n\n# Plot sales by perishability\nlabels = ['Non-Perishable', 'Perishable']\nplt.figure(figsize=(6, 6))\nplt.pie(sales_by_perishability, labels=labels, autopct='%1.1f%%', startangle=90, colors=['lightblue', 'lightcoral'])\nplt.title('Sales Distribution: Perishable vs Non-Perishable Items', fontsize=14)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales data by family and month\ndf['month'] = df['date'].dt.to_period('M')\nmonthly_sales_family = df.groupby(['month', 'family'])['unit_sales'].sum().unstack()\n\n# Plot trends for top families\ntop_families = sales_by_family.head(5).index  # Select top 5 families by sales\nmonthly_sales_family[top_families].plot(figsize=(12, 6))\nplt.title('Monthly Sales Trends for Top Families', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.legend(title='Family', loc='upper left')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Label encode family and class\nle_family = LabelEncoder()\ndf['family_encoded'] = le_family.fit_transform(df['family'])\n\n# Add aggregate features\nfamily_aggregates = df.groupby('family')['unit_sales'].agg(['mean', 'median']).reset_index()\nfamily_aggregates.columns = ['family', 'family_mean_sales', 'family_median_sales']\ndf = pd.merge(df, family_aggregates, on='family', how='left')\n\ndf.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\nprint(oil.shape)\noil.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert 'date' to datetime\noil['date'] = pd.to_datetime(oil['date'])\n\n# Handle missing values by forward-filling\noil['dcoilwtico'].fillna(method='ffill', inplace=True)\n\n# Verify changes\nprint(oil.isnull().sum())\noil.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge oil data with train and test datasets\ndf = pd.merge(df, oil, on='date', how='left')\ntest_df = pd.merge(test_df, oil, on='date', how='left')\n\n# Verify the merge\ndf.head(), test_df.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate correlation between oil prices and sales\ncorrelation = df[['dcoilwtico', 'unit_sales']].corr()\nprint(\"Correlation between oil prices and sales:\")\nprint(correlation)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot oil prices and sales over time\nplt.figure(figsize=(12, 6))\nplt.plot(df['date'], df['dcoilwtico'], label='Oil Price', color='blue')\nplt.plot(df.groupby('date')['unit_sales'].sum(), label='Total Sales', color='green')\nplt.title('Oil Prices vs Sales Trends', fontsize=14)\nplt.xlabel('Date', fontsize=12)\nplt.ylabel('Value', fontsize=12)\nplt.legend()\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group data by month\ndf['month'] = df['date'].dt.to_period('M')\nmonthly_oil_price = oil.groupby(oil['date'].dt.to_period('M'))['dcoilwtico'].mean()\nmonthly_sales = df.groupby('month')['unit_sales'].sum()\n\n# Plot monthly oil prices and sales\nplt.figure(figsize=(12, 6))\nplt.plot(monthly_oil_price.index.astype(str), monthly_oil_price.values, label='Monthly Avg Oil Price', color='blue')\nplt.plot(monthly_sales.index.astype(str), monthly_sales.values, label='Monthly Total Sales', color='green')\nplt.title('Monthly Oil Prices vs Sales', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Value', fontsize=12)\nplt.legend()\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.xticks(rotation=45)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create lagged and rolling features for oil prices\ndf['oil_price_lag1'] = df['dcoilwtico'].shift(1)  # Previous day's oil price\ndf['oil_price_7d_avg'] = df['dcoilwtico'].rolling(window=7).mean()  # 7-day rolling average\ndf['oil_price_30d_avg'] = df['dcoilwtico'].rolling(window=30).mean()  # 30-day rolling average\n\n# Apply the same for the test set\ntest_df['oil_price_lag1'] = test_df['dcoilwtico'].shift(1)\ntest_df['oil_price_7d_avg'] = test_df['dcoilwtico'].rolling(window=7).mean()\ntest_df['oil_price_30d_avg'] = test_df['dcoilwtico'].rolling(window=30).mean()\n\n# Verify the added features\ndf.head(), test_df.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate daily oil price change\ndf['oil_price_pct_change'] = df['dcoilwtico'].pct_change()\n\n# Plot the oil price change\nplt.figure(figsize=(12, 6))\nplt.plot(df['date'], df['oil_price_pct_change'], label='Daily Oil Price Change', color='orange')\nplt.title('Daily Oil Price Change Over Time', fontsize=14)\nplt.xlabel('Date', fontsize=12)\nplt.ylabel('Percentage Change', fontsize=12)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.\n","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge stores.csv with train and test datasets\ndf = pd.merge(df, stores, on='store_nbr', how='left')\ntest_df = pd.merge(test_df, stores, on='store_nbr', how='left')\n\n# Verify the merged data\ndf.head(), test_df.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by city\nsales_by_city = df.groupby('city')['unit_sales'].sum().sort_values(ascending=False)\n\n# Plot sales by city\nplt.figure(figsize=(12, 6))\nsales_by_city.plot(kind='bar', color='skyblue')\nplt.title('Total Sales by City', fontsize=14)\nplt.xlabel('City', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by state\nsales_by_state = df.groupby('state')['unit_sales'].sum().sort_values(ascending=False)\n\n# Plot sales by state\nplt.figure(figsize=(12, 6))\nsales_by_state.plot(kind='bar', color='lightgreen')\nplt.title('Total Sales by State', fontsize=14)\nplt.xlabel('State', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by store type\nsales_by_type = df.groupby('type')['unit_sales'].sum().sort_values(ascending=False)\n\n# Plot sales by store type\nplt.figure(figsize=(8, 6))\nsales_by_type.plot(kind='bar', color='orange')\nplt.title('Total Sales by Store Type', fontsize=14)\nplt.xlabel('Store Type', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group sales by store cluster\nsales_by_cluster = df.groupby('cluster')['unit_sales'].sum().sort_values(ascending=False)\n\n# Plot sales by cluster\nplt.figure(figsize=(10, 6))\nsales_by_cluster.plot(kind='bar', color='purple')\nplt.title('Total Sales by Store Cluster', fontsize=14)\nplt.xlabel('Cluster', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Encode 'type' column\nle_type = LabelEncoder()\ndf['type_encoded'] = le_type.fit_transform(df['type'])\ntest_df['type_encoded'] = le_type.transform(test_df['type'])\n\n# 'cluster' can be used as-is or scaled if necessary\ndf['cluster'] = df['cluster'].astype(int)\ntest_df['cluster'] = test_df['cluster'].astype(int)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compute city-level aggregates\ncity_aggregates = df.groupby('city')['unit_sales'].agg(['mean', 'median']).reset_index()\ncity_aggregates.columns = ['city', 'city_avg_sales', 'city_median_sales']\n\n# Merge city-level aggregates back into the dataset\ndf = pd.merge(df, city_aggregates, on='city', how='left')\ntest_df = pd.merge(test_df, city_aggregates, on='city', how='left')\n\n# Verify the merged data\ndf.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"markdown","source":"Baseline Model","metadata":{}},{"cell_type":"code","source":"from sklearn.dummy import DummyRegressor\nfrom sklearn.metrics import mean_squared_log_error\nimport numpy as np\n\n# Define X and y\nX = df.drop(columns=['unit_sales'])  # Drop target column\ny = df['unit_sales']\n\n# Split data\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Dummy Model\ndummy_model = DummyRegressor(strategy='mean')\ndummy_model.fit(X_train, y_train)\ny_pred = dummy_model.predict(X_test)\n\n# NWRMSLE\nnwrmsle = np.sqrt(mean_squared_log_error(y_test, y_pred))\nprint(\"Baseline Dummy Model NWRMSLE:\", nwrmsle)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import mean_squared_log_error\n\n# Prepare data\nX = df.drop(columns=['unit_sales', 'date'])\ny = df['unit_sales']\n\n# Split data\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# LightGBM model\nlgb_model = lgb.LGBMRegressor(objective='regression', metric='rmse', random_state=42)\nlgb_model.fit(X_train, y_train)\n\n# Predictions and evaluation\ny_pred = lgb_model.predict(X_test)\nnwrmsle = np.sqrt(mean_squared_log_error(y_test, y_pred))\nprint(\"LGBM Model NWRMSLE:\", nwrmsle)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Hyperparameter tuning with GridSearchCV\nparam_grid = {\n    'num_leaves': [31, 50, 70],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'n_estimators': [100, 200, 500]\n}\n\ngrid_search = GridSearchCV(lgb.LGBMRegressor(objective='regression', metric='rmse', random_state=42),\n                           param_grid, cv=3, scoring='neg_root_mean_squared_error', verbose=1)\ngrid_search.fit(X_train, y_train)\n\n# Best model\nbest_lgb_model = grid_search.best_estimator_\nprint(\"Best parameters:\", grid_search.best_params_)\n\n# Evaluate best model\ny_pred = best_lgb_model.predict(X_test)\nnwrmsle = np.sqrt(mean_squared_log_error(y_test, y_pred))\nprint(\"Optimized LGBM Model NWRMSLE:\", nwrmsle)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostRegressor\n\n# CatBoost model\ncat_model = CatBoostRegressor(iterations=500, depth=8, learning_rate=0.1, loss_function='RMSE', verbose=0)\ncat_model.fit(X_train, y_train, cat_features=[X_train.columns.get_loc(c) for c in ['type', 'family', 'locale']])\n\n# Predictions and evaluation\ny_pred = cat_model.predict(X_test)\nnwrmsle = np.sqrt(mean_squared_log_error(y_test, y_pred))\nprint(\"CatBoost Model NWRMSLE:\", nwrmsle)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}