{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.6"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.execute_input":"2024-12-22T18:23:18.807986Z","iopub.status.busy":"2024-12-22T18:23:18.8076Z","iopub.status.idle":"2024-12-22T18:23:20.326948Z","shell.execute_reply":"2024-12-22T18:23:20.325618Z","shell.execute_reply.started":"2024-12-22T18:23:18.807958Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\npaths = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        paths.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-12-22T17:46:04.282894Z","iopub.status.busy":"2024-12-22T17:46:04.282146Z","iopub.status.idle":"2024-12-22T17:46:04.295675Z","shell.execute_reply":"2024-12-22T17:46:04.293918Z","shell.execute_reply.started":"2024-12-22T17:46:04.282842Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.execute_input":"2024-12-22T17:46:07.590664Z","iopub.status.busy":"2024-12-22T17:46:07.590192Z","iopub.status.idle":"2024-12-22T17:46:12.960498Z","shell.execute_reply":"2024-12-22T17:46:12.958739Z","shell.execute_reply.started":"2024-12-22T17:46:07.590631Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfor filenames in paths:\n    with py7zr.SevenZipFile(filenames, mode='r') as z_ref:\n        z_ref.extractall(path='/kaggle/working')","metadata":{"execution":{"iopub.execute_input":"2024-12-22T17:46:12.962641Z","iopub.status.busy":"2024-12-22T17:46:12.962312Z","iopub.status.idle":"2024-12-22T17:47:42.815438Z","shell.execute_reply":"2024-12-22T17:47:42.814119Z","shell.execute_reply.started":"2024-12-22T17:46:12.962613Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Corporación Favorita Grocery Sales Forecasting\n\nPlease read data description: https://www.kaggle.com/c/favorita-grocery-sales-forecasting","metadata":{}},{"cell_type":"markdown","source":"## Splitting Data into Training and Test Sets\nThe data you are working with splited into **training** and **test** sets and stored in the following files:\n\n- **`/kaggle/working/train_data.csv`** – the training dataset used for model training.\n- **`/kaggle/working/test_data.csv`** – the test dataset used to evaluate the model's performance.","metadata":{}},{"cell_type":"code","source":"%%time\ndata = pd.read_csv(\"/kaggle/working/train.csv\")  # убрать точку\ndata[\"date\"] =  pd.to_datetime(data[\"date\"])","metadata":{"execution":{"iopub.execute_input":"2024-12-22T17:52:12.341249Z","iopub.status.busy":"2024-12-22T17:52:12.340416Z","iopub.status.idle":"2024-12-22T17:54:06.816789Z","shell.execute_reply":"2024-12-22T17:54:06.815576Z","shell.execute_reply.started":"2024-12-22T17:52:12.341187Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts_data = data['date'].dt.to_period('M').value_counts().sort_index()\n\nplt.figure(figsize=(12, 6))\nmonthly_counts_data.plot(kind='bar', color='skyblue')\nplt.title('Count of Data for Each Month for All data', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T17:54:31.139966Z","iopub.status.busy":"2024-12-22T17:54:31.139488Z","iopub.status.idle":"2024-12-22T17:54:39.138901Z","shell.execute_reply":"2024-12-22T17:54:39.137605Z","shell.execute_reply.started":"2024-12-22T17:54:31.139933Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nTHRESHOLD_TRAIN_DATE = pd.to_datetime(\"2017-07-01\")\nTHRESHOLD_TEST_DATE = pd.to_datetime(\"2017-08-01\")\n\ndata[data[\"date\"] < THRESHOLD_TRAIN_DATE].to_csv(\"/kaggle/working/train_data.csv\")\ndata[(data[\"date\"] >= THRESHOLD_TRAIN_DATE)&(data[\"date\"] < THRESHOLD_TEST_DATE)].to_csv(\"/kaggle/working/test_data.csv\")\n\n%xdel data","metadata":{"execution":{"iopub.execute_input":"2024-12-22T17:56:35.572574Z","iopub.status.busy":"2024-12-22T17:56:35.571987Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf = pd.read_csv(\"/kaggle/working/train_data.csv\")\ntest_df = pd.read_csv(\"/kaggle/working/test_data.csv\")","metadata":{"execution":{"iopub.execute_input":"2024-12-22T18:16:33.049102Z","iopub.status.busy":"2024-12-22T18:16:33.048621Z","iopub.status.idle":"2024-12-22T18:18:28.157738Z","shell.execute_reply":"2024-12-22T18:18:28.156607Z","shell.execute_reply.started":"2024-12-22T18:16:33.049066Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"date\"] = pd.to_datetime(df[\"date\"])\ntest_df[\"date\"] = pd.to_datetime(test_df[\"date\"])","metadata":{"execution":{"iopub.execute_input":"2024-12-22T18:18:42.428142Z","iopub.status.busy":"2024-12-22T18:18:42.427662Z","iopub.status.idle":"2024-12-22T18:18:54.339639Z","shell.execute_reply":"2024-12-22T18:18:54.338671Z","shell.execute_reply.started":"2024-12-22T18:18:42.428109Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts = df['date'].dt.to_period('M').value_counts().sort_index().reset_index()\nmonthly_counts_test = test_df['date'].dt.to_period('M').value_counts().sort_index().reset_index()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T18:27:51.917192Z","iopub.status.busy":"2024-12-22T18:27:51.916749Z","iopub.status.idle":"2024-12-22T18:27:58.402873Z","shell.execute_reply":"2024-12-22T18:27:58.401884Z","shell.execute_reply.started":"2024-12-22T18:27:51.917162Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"monthly_counts['type'] = 'Train'\nmonthly_counts_test['type'] = 'Test'\ncombined = pd.concat([monthly_counts, monthly_counts_test])\n\nplt.figure(figsize=(12, 6))\nsns.barplot(data=combined, x='date', y='count', hue='type', palette=['blue', 'red'])\nplt.title('Count of Data for Each Month (Train vs Test)', fontsize=14)\nplt.xlabel('Month', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(rotation=45)\nplt.legend(title='Dataset')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T18:32:44.878569Z","iopub.status.busy":"2024-12-22T18:32:44.878068Z","iopub.status.idle":"2024-12-22T18:32:46.06011Z","shell.execute_reply":"2024-12-22T18:32:46.058879Z","shell.execute_reply.started":"2024-12-22T18:32:44.878531Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T18:33:02.687655Z","iopub.status.busy":"2024-12-22T18:33:02.687259Z","iopub.status.idle":"2024-12-22T18:33:02.701078Z","shell.execute_reply":"2024-12-22T18:33:02.699825Z","shell.execute_reply.started":"2024-12-22T18:33:02.687625Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Additional Data\n\n\nTo make more effective use of the data, you also have access to several auxiliary datasets containing contextual information:\n\n#### 1. **`holidays_events.csv`**\n   - **Description**:\n     Contains information about holidays and events that may influence sales.\n   - **Useful Columns**:\n     - `date`: The date of the holiday or event.\n     - `type`: The type of event (national, regional, local).\n     - `locale`: The level of impact (national, regional, local).\n     - `description`: A description of the event.\n   - **How to Use**:\n     - Add a flag indicating whether a date is a holiday to the training and test sets.\n     - Categorize holidays by their `type` and consider their impact on sales.\n     - Use the `locale` column to analyze regional effects.","metadata":{"execution":{"iopub.execute_input":"2024-12-22T19:12:02.319553Z","iopub.status.busy":"2024-12-22T19:12:02.31896Z","iopub.status.idle":"2024-12-22T19:12:02.32548Z","shell.execute_reply":"2024-12-22T19:12:02.323891Z","shell.execute_reply.started":"2024-12-22T19:12:02.319516Z"}}},{"cell_type":"code","source":"holiday_events = pd.read_csv(\"/kaggle/working/holidays_events.csv\")\nprint(holiday_events.shape)\nholiday_events.head()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T19:00:01.436602Z","iopub.status.busy":"2024-12-22T19:00:01.436159Z","iopub.status.idle":"2024-12-22T19:00:01.456663Z","shell.execute_reply":"2024-12-22T19:00:01.455129Z","shell.execute_reply.started":"2024-12-22T19:00:01.436567Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_events = holiday_events[['date', 'type', 'locale']]\nholiday_events['date'] = pd.to_datetime(holiday_events['date'])\nholiday_events['is_national'] = holiday_events['locale'] == 'National'\nholiday_events['is_regional'] = holiday_events['locale'] == 'Regional'\nholiday_events['is_local'] = holiday_events['locale'] == 'Local'\nholiday_events.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_copy = df[['date', 'unit_sales']]\ndf_copy = df_copy.merge(holiday_events[['date', 'type', 'locale', 'is_national', 'is_regional', 'is_local']], on='date', how='left')\ndf_copy['type'].fillna('None', inplace=True)\ndf_copy['locale'].fillna('None', inplace=True)\ndf_copy['is_national'].fillna('None', inplace=True)\ndf_copy['is_regional'].fillna('None', inplace=True)\ndf_copy['is_local'].fillna('None', inplace=True)\n\ndf_copy.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df_copy = test_df[['date', 'unit_sales']]\ntest_df_copy = test_df_copy.merge(holiday_events[['date', 'type', 'locale', 'is_national', 'is_regional', 'is_local']], on='date', how='left')\ntest_df_copy['type'].fillna('None', inplace=True)\ntest_df_copy['locale'].fillna('None', inplace=True)\ntest_df_copy['is_national'].fillna('None', inplace=True)\ntest_df_copy['is_regional'].fillna('None', inplace=True)\ntest_df_copy['is_local'].fillna('None', inplace=True)\n\ntest_df_copy.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_sales = df_copy['unit_sales'].sum()\nholiday_sales = df_copy[(df_copy['type'] == 'Holiday')]['unit_sales'].sum()\nother_sales = total_sales - holiday_sales\nholiday_days = (df_copy['type'] == 'Holiday').sum()\nother_days = df_copy.shape[0] - holiday_days\nprint('Average sales on holidays:', holiday_sales / holiday_days)\nprint('Average sales on other days:', other_sales / other_days)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The average number of sales on holidays is higher than on other days","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_local = df_copy[(df_copy['type'] == 'Holiday') & (df_copy['locale'] == 'Local')]\nlocal_units = holiday_local['unit_sales'].sum()\nlocal_days = len(holiday_local)\n\nholiday_regional = df_copy[(df_copy['type'] == 'Holiday') & (df_copy['locale'] == 'Regional')]\nregional_units = holiday_regional['unit_sales'].sum()\nregional_days = len(holiday_regional)\n\nholiday_national = df_copy[(df_copy['type'] == 'Holiday') & (df_copy['locale'] == 'National')]\nnational_units = holiday_national['unit_sales'].sum()\nnational_days = len(holiday_national)\n\nprint('Average sales on local holidays:', local_units / local_days)\nprint('Average sales on regional holidays:', regional_units / regional_days)\nprint('Average sales on national holidays:', national_units / national_days)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The highest number of sales is on national holidays, then regional.\n# The number of sales on regional holidays is very close to sales on national holidays.\n# On average, local holidays do not increase sales compared to non-holidays.","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 2. **`items.csv`**\n   - **Description**:\n     Contains information about the items being sold.\n   - **Useful Columns**:\n     - `item_nbr`: Unique identifier for each item.\n     - `family`: The category or family the item belongs to (e.g., \"Beverages\").\n     - `class`: A finer categorization of items.\n     - `perishable`: A binary indicator for whether the item is perishable (1 for perishable, 0 otherwise).\n   - **How to Use**:\n     - Use `family` and `class` to group items and analyze trends across categories.\n     - Include the `perishable` indicator to weight predictions for perishable items.","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"/kaggle/working/items.csv\")\nprint(items.shape)\nitems.head()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T19:00:21.140024Z","iopub.status.busy":"2024-12-22T19:00:21.139548Z","iopub.status.idle":"2024-12-22T19:00:21.161814Z","shell.execute_reply":"2024-12-22T19:00:21.160088Z","shell.execute_reply.started":"2024-12-22T19:00:21.139993Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_items = df[['item_nbr', 'unit_sales']]\ndf_items = df_items.merge(items, on='item_nbr', how='left')\ndf_items.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"grouped_sales = df_items.groupby(['family', 'class'])['unit_sales'].sum().reset_index()\ngrouped_sales = grouped_sales.sort_values(by='unit_sales', ascending=False)\ngrouped_sales.head(10)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_items['weighted_sales'] = df_items['unit_sales'] * (1 + df_items['perishable'] * 0.25)\ngrouped_weighted_sales = df_items.groupby(['family', 'class'])['weighted_sales'].sum().reset_index()\ngrouped_weighted_sales = grouped_weighted_sales.sort_values(by='weighted_sales', ascending=False)\ngrouped_weighted_sales.head(10)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. **`oil.csv`**\n   - **Description**:\n     Contains daily oil price data, which can serve as an external economic indicator.\n   - **Useful Columns**:\n     - `date`: The date corresponding to the oil price.\n     - `dcoilwtico`: The daily oil price.\n   - **How to Use**:\n     - Join this dataset with the training and test sets on the `date` column to include oil price as a feature.\n     - Analyze the correlation between oil price fluctuations and sales trends.","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv(\"/kaggle/working/oil.csv\")\noil['date'] = pd.to_datetime(oil['date'])\nprint(oil.shape)\noil.head()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T19:00:42.115057Z","iopub.status.busy":"2024-12-22T19:00:42.114519Z","iopub.status.idle":"2024-12-22T19:00:42.134103Z","shell.execute_reply":"2024-12-22T19:00:42.132379Z","shell.execute_reply.started":"2024-12-22T19:00:42.115018Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_oil = df[['date', 'unit_sales']]\ndf_oil = df_oil.merge(oil, on='date', how='left')\ndf_oil['dcoilwtico'].fillna('None', inplace=True)\ndf_oil['dcoilwtico'] = df_oil['dcoilwtico'].fillna(method='ffill')\ndf_oil.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_oil['dcoilwtico'] = pd.to_numeric(df_oil['dcoilwtico'], errors='coerce')\nsales_by_date = df_oil.groupby('date').agg({'unit_sales': 'sum', 'dcoilwtico': 'mean'}).reset_index()\ncorrelation = sales_by_date['unit_sales'].corr(sales_by_date['dcoilwtico'])\nprint(f'Correlation between oil prices and unit sales: {correlation}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The correlation coefficient of approximately -0.701 \n# indicates a strong negative correlation between oil prices and sales. \n# This means that as oil prices increase, sales is likely to decrease. ","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df_oil = test_df[['date', 'unit_sales']]\ntest_df_oil = test_df_oil.merge(oil, on='date', how='left')\ntest_df_oil['dcoilwtico'].fillna('None', inplace=True)\ntest_df_oil['dcoilwtico'] = test_df_oil['dcoilwtico'].fillna(method='ffill')\ntest_df_oil.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df_oil['dcoilwtico'] = pd.to_numeric(test_df_oil['dcoilwtico'], errors='coerce')\nsales_by_date = test_df_oil.groupby('date').agg({'unit_sales': 'sum', 'dcoilwtico': 'mean'}).reset_index()\ncorrelation = sales_by_date['unit_sales'].corr(sales_by_date['dcoilwtico'])\nprint(f'Correlation between oil prices and unit sales: {correlation}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# A correlation coefficient 0.067 indicates a very weak positive correlation between oil prices and sales trends","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 4. **`stores.csv`**\n   - **Description**:\n     Contains information about the stores where items are sold.\n   - **Useful Columns**:\n     - `store_nbr`: Unique identifier for each store.\n     - `city`: The city where the store is located.\n     - `state`: The state where the store is located.\n     - `type`: The store type.\n     - `cluster`: A grouping of similar stores based on sales patterns.\n   - **How to Use**:\n     - Use `city` and `state` for regional analysis of sales trends.\n     - Incorporate `type` and `cluster` to identify patterns and group stores with similar behavior.\n","metadata":{}},{"cell_type":"code","source":"stores = pd.read_csv(\"/kaggle/working/stores.csv\")\nprint(stores.shape)\nstores.head()","metadata":{"execution":{"iopub.execute_input":"2024-12-22T19:01:16.159807Z","iopub.status.busy":"2024-12-22T19:01:16.159314Z","iopub.status.idle":"2024-12-22T19:01:16.176923Z","shell.execute_reply":"2024-12-22T19:01:16.175827Z","shell.execute_reply.started":"2024-12-22T19:01:16.15977Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_stores = df[['store_nbr', 'unit_sales']]\ndf_stores = df_stores.merge(stores, on='store_nbr', how='left')\nregional_sales_trends = df_stores.groupby(['city', 'state'])['unit_sales'].sum().reset_index()\nsorted_regional_sales_trends = regional_sales_trends.sort_values(by='unit_sales', ascending=False).reset_index()\nsorted_regional_sales_trends","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pattern_analysis = df_stores.groupby(['type', 'cluster'])['unit_sales'].agg(['sum', 'mean', 'count']).reset_index()\npattern_analysis","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.barplot(data=pattern_analysis, x='cluster', y='sum', hue='type')\nplt.title('Total Sales by Store Type and Cluster', fontsize=14)\nplt.xlabel('Cluster', fontsize=12)\nplt.ylabel('Total Sales', fontsize=12)\nplt.legend(title='Store Type', fontsize=12)\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"markdown","source":"**[READ DATA DESCRIPTION CAREFULY](https://www.kaggle.com/competitions/favorita-grocery-sales-forecasting) !!!!** \n\n## Exploratory Data Analysis and Model Training\n\n### 1. Baseline Model\n- Consider which model would be suitable as `DummyModel` for this task, then create, train, and use its predictions as a baseline.\n\n### 2. Feature Engineering\n- Generate new features by incorporating data from additional datasets to enhance the model's predictive capabilities.\n\n### 3. Model Training and Optimization\n- Utilize advanced regression models, such as **LGBM Regression** or **CatBoost Regression**, to achieve the best results. Focus on hyperparameter tuning and evaluation metrics to optimize performance. **Use foremention Normalized Weighted Root Mean Squared Logarithmic Error**.  \n\n### Thats All :-)\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.model_selection import train_test_split\nfrom lightgbm import LGBMRegressor\n\ndef nwrmsle(y_true, y_pred, weights):\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    weights = np.array(weights)\n    \n    mask = ~(np.isnan(y_true) | np.isnan(y_pred) | np.isnan(weights))\n    y_true = y_true[mask]\n    y_pred = y_pred[mask]\n    weights = weights[mask]\n    \n    if len(y_true) == 0:\n        return np.nan\n    \n    y_true = np.maximum(y_true, 0)\n    y_pred = np.maximum(y_pred, 0)\n    \n    epsilon = 1e-7\n    y_true = y_true + epsilon\n    y_pred = y_pred + epsilon\n    \n    sle = (np.log1p(y_pred) - np.log1p(y_true)) ** 2\n    wsle = weights * sle\n    \n    return np.sqrt(np.mean(wsle))\n\ndef load_data():\n    dtypes = {\n        'store_nbr': 'int8',\n        'item_nbr': 'int32',\n        'unit_sales': 'float32'\n    }\n    \n    train = pd.read_csv('/kaggle/working/train_data.csv', dtype=dtypes)\n    train['onpromotion'] = train['onpromotion'].fillna(False).astype(bool)\n    test = pd.read_csv('/kaggle/working/test_data.csv', dtype=dtypes)\n    test['onpromotion'] = test['onpromotion'].fillna(False).astype(bool)\n    stores = pd.read_csv('/kaggle/working/stores.csv', dtype={'store_nbr': 'int8'})\n    items = pd.read_csv('/kaggle/working/items.csv', dtype={'item_nbr': 'int32', 'perishable': 'int8'})\n    transactions = pd.read_csv('/kaggle/working/transactions.csv', dtype={'store_nbr': 'int8', 'transactions': 'float32'})\n    oil = pd.read_csv('/kaggle/working/oil.csv', dtype={'dcoilwtico': 'float32'})\n    holidays = pd.read_csv('/kaggle/working/holidays_events.csv')\n    \n    date_cols = ['date']\n    for df in [train, test, transactions, oil, holidays]:\n        for col in date_cols:\n            if col in df.columns:\n                df[col] = pd.to_datetime(df[col])\n    \n    return train, test, stores, items, transactions, oil, holidays\n\ndef create_features(df, stores, items, transactions, oil, holidays):\n    chunk_size = 1000000\n    result_dfs = []\n    \n    for start_idx in range(0, len(df), chunk_size):\n        end_idx = min(start_idx + chunk_size, len(df))\n        chunk = df.iloc[start_idx:end_idx].copy()\n        \n        chunk = chunk.merge(stores, on='store_nbr', how='left')\n        chunk = chunk.merge(items, on='item_nbr', how='left')\n        \n        chunk['year'] = chunk['date'].dt.year\n        chunk['month'] = chunk['date'].dt.month\n        chunk['day'] = chunk['date'].dt.day\n        chunk['dayofweek'] = chunk['date'].dt.dayofweek\n        chunk['is_weekend'] = chunk['dayofweek'].isin([5,6]).astype('int8')\n        \n        chunk = chunk.merge(transactions, on=['date', 'store_nbr'], how='left')\n        \n        chunk = chunk.merge(oil, on='date', how='left')\n        chunk['dcoilwtico'] = chunk['dcoilwtico'].fillna(method='ffill')\n        \n        chunk['is_holiday'] = chunk['date'].isin(\n            holidays[holidays['type'] != 'Work Day']['date']\n        ).astype('int8')\n        \n        if 'onpromotion' in chunk.columns:\n            chunk['onpromotion'] = chunk['onpromotion'].fillna(False).astype('int8')\n        \n        result_dfs.append(chunk)\n        \n    return pd.concat(result_dfs, ignore_index=True)\n\ndef create_baseline(X_train, y_train):\n    dummy = DummyRegressor(strategy='mean')\n    dummy.fit(X_train, y_train)\n    return dummy\n\ndef create_lgbm():\n    params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'num_leaves': 31,\n        'learning_rate': 0.05,\n        'feature_fraction': 0.9,\n        'bagging_fraction': 0.8,\n        'bagging_freq': 5,\n        'verbose': -1\n    }\n    return LGBMRegressor(**params)\n\ndef main():\n    train, test, stores, items, transactions, oil, holidays = load_data()\n    \n    feature_cols = ['store_nbr', 'item_nbr', 'onpromotion', 'year', 'month', \n                   'day', 'dayofweek', 'is_weekend', 'transactions', \n                   'dcoilwtico', 'is_holiday']\n    \n    train_processed = create_features(train, stores, items, transactions, oil, holidays)\n\n    for col in feature_cols:\n        if train_processed[col].isna().any():\n            if train_processed[col].dtype in ['int8', 'int32', 'int64']:\n                train_processed[col] = train_processed[col].fillna(0)\n            else:\n                train_processed[col] = train_processed[col].fillna(train_processed[col].mean())\n    \n    X = train_processed[feature_cols]\n    y = train_processed['unit_sales']\n    weights = np.where(train_processed['perishable'] == 1, 1.25, 1.0)\n    \n    X_train, X_val, y_train, y_val, w_train, w_val = train_test_split(\n        X, y, weights, test_size=0.2, random_state=42\n    )\n    \n    baseline = create_baseline(X_train, y_train)\n    baseline_pred = baseline.predict(X_val)\n    baseline_score = nwrmsle(y_val, baseline_pred, w_val)\n    print(f\"Baseline NWRMSLE: {baseline_score:.4f}\")\n    \n    lgbm = create_lgbm()\n    lgbm.fit(X_train, y_train, sample_weight=w_train)\n    lgbm_pred = lgbm.predict(X_val)\n    lgbm_score = nwrmsle(y_val, lgbm_pred, w_val)\n    print(f\"LGBM NWRMSLE: {lgbm_score:.4f}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{},"outputs":[],"execution_count":null}]}