{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5260,"databundleVersionId":37827,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-07T14:15:33.892626Z","iopub.execute_input":"2024-07-07T14:15:33.892919Z","iopub.status.idle":"2024-07-07T14:15:34.892525Z","shell.execute_reply.started":"2024-07-07T14:15:33.892895Z","shell.execute_reply":"2024-07-07T14:15:34.891426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

Introduction

\n\nGrupo Bimbo, a leading multinational bakery company, faces a unique challenge in managing the inventory of its products. With a typical shelf life of just one week, the accuracy of daily inventory calculations is paramount. Currently, these calculations are performed by direct delivery sales employees who rely on their personal experiences to predict the forces of supply, demand, and consumer behavior at each store. The margin for error in this process is minimal. Underestimating demand results in empty shelves and lost sales, while overestimating demand leads to excess product returns and increased expenses.\n\nGrupo Bimbo aims to create a predictive model that can accurately forecast inventory needs based on historical data, thereby optimizing the supply chain and improving efficiency.","metadata":{}},{"cell_type":"markdown","source":"

Approach

\n\n\n- [1.1 Import Libraries](#1.1)\n- [1.2 Configuration](#1.2)\n- [1.3 Important Functions](#1.3)\n- [1.4 Import Data](#1.4)\n- [1.5 Quick overview](#1.5)\n- [2. Exploratory Data Analysis ](#2)\n- [3.1 Feature Engineering](#3.1)\n- [3.2 Missing Data Handling](#3.2)\n- [3.3 Outlier Handling](#3.3)\n- [3.4 Feature Transformation](#3.4)\n- [3.5 Feature Creation](#3.5)\n- [3.6 Feature Selection](#3.6)\n- [4. Data Pipeline](#4)\n- [4.1 Pipeline Creation](#4.1)\n- [4.2 Visualizing the Pipeline ](#4.2)\n- [5. Model Building](#5)\n- [5.1 Train Test Split](#5.1)\n- [5.2 Training the Model](#5.2)\n- [5.3 Performance Evaluation](#5.3)\n- [5.4 Ensembling for Performance Improvement](#5.4)\n- [5.5 Model Explainability](#5.5)\n- [6. Model Infrence](#6)\n- [6.1 Loading the Model](#6.1)\n- [6.2 Data Loading and Prepration](#6.2)\n- [6.3 Model Forecasting](#6.3)","metadata":{}},{"cell_type":"markdown","source":"\n## 1.1 Import Libraries ","metadata":{}},{"cell_type":"code","source":"!pip3 install -q dagshub\n!pip3 install -q mlflow","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-07-07T14:15:34.894411Z","iopub.execute_input":"2024-07-07T14:15:34.894906Z","iopub.status.idle":"2024-07-07T14:16:15.140234Z","shell.execute_reply.started":"2024-07-07T14:15:34.894871Z","shell.execute_reply":"2024-07-07T14:16:15.139109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# System related tasks\nimport os\nimport gc\nimport warnings\n\n# Data manipulation and numerical operations\nimport numpy as np \nimport pandas as pd \nimport polars as pl\n\n# Data visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport plotly.express as px\n\n# Scikit-learn model selection and evaluation\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, TimeSeriesSplit\nfrom sklearn.metrics import mean_squared_error, r2_score, roc_auc_score, accuracy_score\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.base import BaseEstimator, TransformerMixin, clone\n\n# Machine learning models\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier, VotingRegressor\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier\n\n# Gradient boosting frameworks\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier, CatBoostRegressor\n\n# Optimization and utilities\nimport optuna\nimport joblib\nfrom kaggle_secrets import UserSecretsClient\n\n# For Feature Selection\nfrom sklearn.feature_selection import SelectKBest, chi2\n\n# Experiment tracking and collaboration\nimport dagshub\nimport mlflow\n\n# Model Explainability \nimport shap\n\n# Ensure warnings are ignored and plots are displayed inline\nwarnings.filterwarnings('ignore')\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-07-07T14:16:15.142447Z","iopub.execute_input":"2024-07-07T14:16:15.143117Z","iopub.status.idle":"2024-07-07T14:16:26.583098Z","shell.execute_reply.started":"2024-07-07T14:16:15.143074Z","shell.execute_reply":"2024-07-07T14:16:26.582326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 1.2 Configuration ","metadata":{}},{"cell_type":"code","source":"class CFG:\n \"\"\"\n Configuration class for parameters, models, paths, and cross-validation strategies.\n Please use caps lock for variable names while filling in parameters.\n \"\"\"\n # Paths\n BASE_PATH = \"/kaggle/input/grupo-bimbo-inventory-demand\"\n TRAIN_PATH = f\"{BASE_PATH}/train.csv.zip\"\n TEST_PATH = f\"{BASE_PATH}/test.csv.zip\"\n \n # Columns Rename to ensure names are readable and relevant\n COLUMN_RENAME = {\n 'Semana': 'Week', \n 'Agencia_ID': 'Sales_Depot_ID', \n 'Canal_ID': 'Sales_Channel_ID', \n 'Ruta_SAK': 'Route_ID', \n 'Cliente_ID': 'Client_ID',\n 'Producto_ID': 'Product_ID', \n 'Venta_uni_hoy': 'Sales_unit_this_week', \n 'Venta_hoy': 'Sales_this_week', \n 'Dev_uni_proxima': 'Returns_unit_next_week',\n 'Dev_proxima': 'Returns_next_week', \n 'Demanda_uni_equil': 'Adjusted_Demand', \n 'NombreCliente': 'Client_name', \n 'NombreProducto': 'Product_Name',\n 'Town': 'Town', \n 'State': 'State'\n }\n \n # Data Type Casting Settings to ensure smoth pipeline operation\n DATA_TYPES = {\n 'Week': 'float64',\n 'Sales_Depot_ID': 'category',\n 'Sales_Channel_ID': 'category',\n 'Route_ID': 'category',\n 'Client_ID': 'category',\n 'Product_ID': 'category',\n 'Sales_unit_this_week': 'int64',\n 'Sales_this_week': 'int64',\n 'Returns_unit_next_week': 'int64',\n 'Adjusted_Demand': 'int64'\n }\n # Selecting Categorical variable. Client_ID is excluded due to high dimensionality\n \n CATEGORICAL_VARIABLES = ('Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Product_ID')\n # Defining Target Variable \n \n TARGET_VARIABLE = 'Adjusted_Demand'\n \n # Model Settings to train the model with hyperparameters\n MODELS = {\n \"xgboost\": {\n \"model\": xgb.XGBRegressor,\n \"params\": {\n \"n_estimators\": 100,\n \"max_depth\": 8,\n \"learning_rate\": 0.1,\n \"subsample\": 0.8,\n \"colsample_bytree\": 0.8,\n \"random_state\": 42,\n \"tree_method\": \"gpu_hist\", \n \"enable_categorical\": True,\n \"gpu_id\": 0 \n }\n },\n \"catboost\": {\n \"model\": CatBoostRegressor,\n \"params\": {\n \"iterations\": 100,\n \"learning_rate\": 0.1,\n \"depth\": 8,\n \"verbose\": False,\n \"task_type\": \"GPU\", \n \"devices\": '0:1' \n }\n },\n \"lightgbm\": {\n \"model\": lgb.LGBMRegressor,\n \"params\": {\n \"n_estimators\": 50,\n \"learning_rate\": 0.1,\n \"max_depth\": 8,\n \"subsample\": 0.8,\n \"colsample_bytree\": 0.8,\n \"random_state\": 42,\n \"device\": \"cpu\", \n \"gpu_device_id\": 0,\n \"max_bin\": 16, \n \"num_leaves\": 16,\n \"verbosity\": -1 \n }\n }\n }\n\n \n # Cross-Validation Settings for timeseries problem\n CV_STRATEGY = {TimeSeriesSplit(n_splits=2)\n }\n \n # MLflow Settings to track the performance of model training\n MLFLOW_TRACKING_URI = \"https://dagshub.com/agshiv92/GrupoBimbo_InventoryDemand_Solutions.mlflow\"\n MLFLOW_EXPERIMENT_NAME = \"Group_Bimbo_Predictive_Model\"\n DAGSHUB_REPO_OWNER = \"agshiv92\"\n DAGSHUB_REPO = \"GrupoBimbo_InventoryDemand_Solutions\"\n MLFLOW_URI = \"https://dagshub.com/username/repository_name.mlflow\"\n MLFLOW_LOCAL_URI = \"/kaggle/working/mlflow\"\n # Setting seeed to ensure reproducibillity\n SEED=42\n # Model Save Path \n MODEL_SAVE_PATH = '/kaggle/working/models'\n EXPERIMENT_RECORDING = 'LOCAL'\n ","metadata":{"execution":{"iopub.status.busy":"2024-07-07T14:16:26.585772Z","iopub.execute_input":"2024-07-07T14:16:26.586715Z","iopub.status.idle":"2024-07-07T14:16:26.599551Z","shell.execute_reply.started":"2024-07-07T14:16:26.586685Z","shell.execute_reply":"2024-07-07T14:16:26.598722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 1.3 Important Functions ","metadata":{}},{"cell_type":"code","source":"# Defining some important function which will be used for the analysis of data\ndef inspect_columns(df, df_name):\n \"\"\"A helper function that does a better job than df.info() and df.describe()\"\"\"\n \n total_rows = len(df)\n result = pd.DataFrame({\n 'total_rows': [total_rows] * df.shape[1],\n 'rows_with_missing_values': df.isnull().sum(),\n 'unique': df.nunique() == total_rows,\n 'cardinality': df.nunique(),\n 'with_null': df.isna().any(),\n 'null_pct': round((df.isnull().sum() / total_rows) * 100, 2),\n '1st_row': df.iloc[0],\n 'random_row': df.iloc[np.random.randint(low=0, high=total_rows)],\n 'last_row': df.iloc[-1],\n 'dtype': df.dtypes,\n })\n \n # Print the name of the dataframe\n print(f\"\\n{'='*10} {df_name} {'='*10}\\n\")\n \n # Print the head of the dataframe\n print(\"First few rows of the dataframe:\\n\")\n display(df.head())\n \n # Print the resulting statistics\n print(\"Detailed statistics:\\n\")\n display(result)\n \ndef cast_data_types(df, data_types):\n \"\"\"A helper function to cast the pre determined data type for the columns in dataframe\"\"\"\n for column,data_type in data_types.items():\n if column in df.columns:\n df[column] = df[column].astype(data_type)\n return df\n\ndef get_or_create_experiment_id(name):\n \"\"\"Function to create or get the experiment id\"\"\"\n exp = mlflow.get_experiment_by_name(name)\n if exp is None:\n exp_id = mlflow.create_experiment(name)\n return exp_id\n return exp.experiment_id\n\ndef reduce_memory_usage(df):\n \"\"\"Reduce memory usage of a dataframe by downcasting data types.\"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n# print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n \n for col in df.columns:\n col_type = df[col].dtype\n # Skip timestamp columns\n if pd.api.types.is_datetime64_any_dtype(df[col]):\n continue\n if (col_type != object) & (col_type != object):\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n else:\n df[col] = df[col].astype('category')\n \n end_mem = df.memory_usage().sum() / 1024**2\n# print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n print(f'Decreased by {100 * (start_mem - end_mem) / start_mem:.1f}%')\n return df\n\ndef check_and_create_directory(directory_path):\n \"\"\" To create directory if not exist\"\"\"\n if not os.path.exists(directory_path):\n os.makedirs(directory_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-07T14:16:26.601205Z","iopub.execute_input":"2024-07-07T14:16:26.601778Z","iopub.status.idle":"2024-07-07T14:16:26.633374Z","shell.execute_reply.started":"2024-07-07T14:16:26.601752Z","shell.execute_reply":"2024-07-07T14:16:26.632480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 1.4 Import Data ","metadata":{}},{"cell_type":"code","source":"# Load datasets\ntrain_df = pd.read_csv(CFG.TRAIN_PATH, compression='zip')","metadata":{"execution":{"iopub.status.busy":"2024-07-07T14:16:26.634575Z","iopub.execute_input":"2024-07-07T14:16:26.635153Z","iopub.status.idle":"2024-07-07T14:17:49.676145Z","shell.execute_reply.started":"2024-07-07T14:16:26.635122Z","shell.execute_reply":"2024-07-07T14:17:49.675225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## 1.5 Quick overview ","metadata":{}},{"cell_type":"code","source":"inspect_columns(train_df,\"Training Dataframe\")","metadata":{"execution":{"iopub.status.busy":"2024-07-07T14:17:49.677405Z","iopub.execute_input":"2024-07-07T14:17:49.677719Z","iopub.status.idle":"2024-07-07T14:18:00.653871Z","shell.execute_reply.started":"2024-07-07T14:17:49.677694Z","shell.execute_reply":"2024-07-07T14:18:00.652988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n#### About Data:\n \nThe dataset provided about Grupo Bimbo. It consists of 7 million records, capturing weekly sales transactions and returns.\n\n#### Columns Description\n\n- **Semana**: Week number (From Thursday to Wednesday). This temporal feature helps identify seasonal and weekly trends in sales and demand.\n- **Agencia_ID**: Sales Depot ID. Each depot serves as a distribution point for products and is identified by a unique ID.\n- **Canal_ID**: Sales Channel ID. Represents the sales channel through which products are sold, such as retail, wholesale, etc.\n- **Ruta_SAK**: Route ID. Indicates the delivery route for sales depots. Multiple routes can exist within a single sales depot.\n- **Cliente_ID**: Client ID. Unique identifier for each client.\n- **NombreCliente**: Client name. Descriptive name of the client associated with the Client ID.\n- **Producto_ID**: Product ID. Unique identifier for each product.\n- **NombreProducto**: Product Name. Descriptive name of the product associated with the Product ID.\n- **Venta_uni_hoy**: Sales units this week. Represents the number of units sold for a given product within the week.\n- **Venta_hoy**: Sales this week in pesos. Represents the total sales in monetary units for a given product within the week.\n- **Dev_uni_proxima**: Returns units next week. Indicates the number of units returned for a given product in the following week.\n- **Dev_proxima**: Returns next week in pesos. Represents the total monetary value of returned products in the following week.\n- **Demanda_uni_equil**: Target variable for prediction, representing the adjusted demand for a given product.","metadata":{}},{"cell_type":"markdown","source":"\n\nBack to top\n\n## 2 Exploratory Data Analysis ","metadata":{}},{"cell_type":"markdown","source":"EDA for this is available here https://www.kaggle.com/code/agshiv92/group-bimbo-eda","metadata":{}},{"cell_type":"markdown","source":"\n## 3 Feature Engineering \n### 3.1 Missing Data Handling ","metadata":{}},{"cell_type":"markdown","source":"There is no missing data in the dataframe. If there are missing data there are there are below methods to handle missing data. \n\n### Handling Missing Data\n\n1. **Remove Missing Data**\n - Remove rows with missing values\n - Remove columns with missing values\n\n2. **Impute Missing Data**\n - **Mean Imputation**: Replace missing values with the mean.\n - **Median Imputation**: Replace missing values with the median.\n - **Mode Imputation**: Replace missing values with the mode.\n - **Forward Fill**: Replace missing values with the previous value.\n - **Backward Fill**: Replace missing values with the next value.\n - **Interpolation**: Use linear interpolation to estimate missing values.\n - **K-Nearest Neighbors (KNN) Imputation**: Use the nearest neighbors to impute missing values.\n - **Predictive Imputation**: Use regression or other models to predict missing values.\n","metadata":{}},{"cell_type":"markdown","source":"\n### 3.2 Outlier Handling ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\ntrain_df.boxplot(column='Demanda_uni_equil', by='Semana', grid=False, showfliers=False, patch_artist=True)\nplt.title('Box Plot of Adjusted Demand by Week')\nplt.suptitle('') \nplt.xlabel('Semana')\nplt.ylabel('Adjusted Demand')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-07T14:19:26.890294Z","iopub.execute_input":"2024-07-07T14:19:26.890663Z","iopub.status.idle":"2024-07-07T14:21:22.775178Z","shell.execute_reply.started":"2024-07-07T14:19:26.890638Z","shell.execute_reply":"2024-07-07T14:21:22.774284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 3.3 Feature Transformation ","metadata":{}},{"cell_type":"markdown","source":"#### 3.3.0 Data Prepration\nData prepration by giving appropriate column name and converting the data in appropriate data type","metadata":{}},{"cell_type":"code","source":"def prepare_data(df):\n df = cast_data_types(df, CFG.DATA_TYPES)\n df = df.rename(CFG.COLUMN_RENAME,axis=1)\n if 'Adjusted_Demand' not in df.columns: # To take care of Prediction\n df['Adjusted_Demand']=0\n #Assuming Week 1 starts from 1st January 2024\n if 'Week' in df.columns:\n df['Week'] = pd.to_datetime('2024-01-01') + pd.to_timedelta(df['Week'].astype(int) * 7, unit='D')\n df = reduce_memory_usage(df)\n return df\ntrain_df = prepare_data(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:38:07.550355Z","iopub.execute_input":"2024-07-07T08:38:07.550648Z","iopub.status.idle":"2024-07-07T08:38:18.249875Z","shell.execute_reply.started":"2024-07-07T08:38:07.550623Z","shell.execute_reply":"2024-07-07T08:38:18.248923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3.3.1 One Hot Encoding\n\nI have chosen to avoid one-hot encoding for categorical variables. Instead, I will be leveraging advanced machine learning algorithms like LightGBM, which natively support categorical data. This decision is driven by three key factors:- Efficiency, Performance and \nSimplicity","metadata":{}},{"cell_type":"markdown","source":"#### 3.3.2 Lagged Features\nLagged features are the demand in previouse week for the same product and client. This feature is helpful to understand the dempendency of the demand on previous history","metadata":{}},{"cell_type":"code","source":"def lagged_feature_creation_demand(df):\n df = df.sort_values(by=['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID', 'Week'])\n df['Lagged_Demand_1'] = df.groupby(['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID'])['Adjusted_Demand'].shift(1)\n df['Lagged_Demand_2'] = df.groupby(['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID'])['Adjusted_Demand'].shift(2)\n df['Lagged_Demand_3'] = df.groupby(['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID'])['Adjusted_Demand'].shift(3)\n df = reduce_memory_usage(df)\n gc.collect() \n return df\ntrain_df = lagged_feature_creation_demand(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:38:18.251324Z","iopub.execute_input":"2024-07-07T08:38:18.252132Z","iopub.status.idle":"2024-07-07T08:39:27.305305Z","shell.execute_reply.started":"2024-07-07T08:38:18.252097Z","shell.execute_reply":"2024-07-07T08:39:27.304324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lagged_feature_creation_return(df):\n df = df.sort_values(by=['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID', 'Week'])\n df['Lagged_Return_1'] = df.groupby(['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID'])['Returns_unit_next_week'].shift(1)\n df['Lagged_Return_2'] = df.groupby(['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID'])['Returns_unit_next_week'].shift(2)\n df['Lagged_Return_3'] = df.groupby(['Sales_Depot_ID', 'Sales_Channel_ID', 'Route_ID', 'Client_ID', 'Product_ID'])['Returns_unit_next_week'].shift(3)\n df = reduce_memory_usage(df)\n gc.collect()\n return df\ntrain_df = lagged_feature_creation_return(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:39:27.308642Z","iopub.execute_input":"2024-07-07T08:39:27.308960Z","iopub.status.idle":"2024-07-07T08:40:37.921849Z","shell.execute_reply.started":"2024-07-07T08:39:27.308934Z","shell.execute_reply":"2024-07-07T08:40:37.920941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3.3.3 Rolling Features\nRolling features capture the aggregated values over a specific window of time, such as the average or sum of demand over the past few weeks. These features are instrumental in understanding trends and patterns in the data.\n","metadata":{}},{"cell_type":"code","source":"# Convert to Polars DataFrame\ndef rolling_feature_creation(df, window):\n window_size = window\n df_pl = pl.DataFrame(df.reset_index())\n df_pl = df_pl.with_columns([pl.col(\"Adjusted_Demand\").cast(pl.Float32)])\n df_pl_lazy = df_pl.lazy()\n # Apply rolling calculations\n df_pl_lazy = df_pl_lazy.with_columns([\n pl.col(\"Lagged_Demand_1\").rolling_mean(window_size).over([\"Sales_Depot_ID\", \"Sales_Channel_ID\", \"Route_ID\", \"Client_ID\", \"Product_ID\"]).alias(\"Rolling_Mean_3\"),\n pl.col(\"Lagged_Demand_1\").rolling_sum(window_size).over([\"Sales_Depot_ID\", \"Sales_Channel_ID\", \"Route_ID\", \"Client_ID\", \"Product_ID\"]).alias(\"Rolling_Sum_3\"),\n pl.col(\"Lagged_Demand_1\").rolling_std(window_size).over([\"Sales_Depot_ID\", \"Sales_Channel_ID\", \"Route_ID\", \"Client_ID\", \"Product_ID\"]).alias(\"Rolling_Std_3\")\n ])\n # Collect the result\n df_pl_optimized = df_pl_lazy.collect()\n df = df_pl_optimized.to_pandas()\n \n # Clean up\n del df_pl_optimized\n del df_pl\n del df_pl_lazy\n gc.collect()\n # Reduce memory usage and fill NaN values\n df = reduce_memory_usage(df)\n gc.collect()\n df = df.fillna(0)\n return df\ntrain_df = rolling_feature_creation(train_df,3)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:40:37.923091Z","iopub.execute_input":"2024-07-07T08:40:37.923383Z","iopub.status.idle":"2024-07-07T08:42:53.120623Z","shell.execute_reply.started":"2024-07-07T08:40:37.923358Z","shell.execute_reply":"2024-07-07T08:42:53.119659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 3.4 Feature Creation \n\n#### 3.4.1 Sales Growth","metadata":{}},{"cell_type":"code","source":"def sales_growth(df):\n df['Sales_Growth_Rate_1'] = (df['Lagged_Demand_1'] - df['Lagged_Demand_2']) / df['Lagged_Demand_1']\n df['Sales_Growth_Rate_2'] = (df['Lagged_Demand_2'] - df['Lagged_Demand_3']) / df['Lagged_Demand_2']\n return df\ntrain_df = sales_growth(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:42:53.121799Z","iopub.execute_input":"2024-07-07T08:42:53.122108Z","iopub.status.idle":"2024-07-07T08:42:57.291891Z","shell.execute_reply.started":"2024-07-07T08:42:53.122082Z","shell.execute_reply":"2024-07-07T08:42:57.290887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3.4.2 Client Total Sales","metadata":{}},{"cell_type":"code","source":"def client_total_sales(df):\n client_sales = df.groupby(['Client_ID','Week'])['Lagged_Demand_1'].sum().reset_index().rename(columns={'Lagged_Demand_1': 'Total_Sales_per_Client'}).copy()\n df = df.merge(client_sales, on=['Client_ID','Week'], how='left')\n return df\ntrain_df = client_total_sales(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:42:57.293184Z","iopub.execute_input":"2024-07-07T08:42:57.293475Z","iopub.status.idle":"2024-07-07T08:43:14.138334Z","shell.execute_reply.started":"2024-07-07T08:42:57.293452Z","shell.execute_reply":"2024-07-07T08:43:14.137197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def final_processing(df,categorical_vairable):\n df = df.replace([np.inf, -np.inf], np.nan) # Replacing infinite values with Nan\n df = df.fillna(0) # Filling na with 0\n df.set_index('Week', inplace=True)\n for col in categorical_vairable:\n df[col] = df[col].astype('category') \n df = df.drop(['index','Sales_unit_this_week', 'Sales_this_week','Returns_unit_next_week', 'Returns_next_week'],axis=1)\n return df\ntrain_df = final_processing(train_df,CFG.CATEGORICAL_VARIABLES)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:43:14.139691Z","iopub.execute_input":"2024-07-07T08:43:14.140018Z","iopub.status.idle":"2024-07-07T08:43:38.713781Z","shell.execute_reply.started":"2024-07-07T08:43:14.139993Z","shell.execute_reply":"2024-07-07T08:43:38.712992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 3.5 Feature Selection ","metadata":{}},{"cell_type":"code","source":"numeric_columns = [col for col in train_df.columns if col not in CFG.CATEGORICAL_VARIABLES]\ncorr_matrix = train_df[numeric_columns].corr()\n# Plot the heatmap\nplt.figure(figsize=(12, 8))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm')\nplt.title('Correlation Matrix with Heatmap')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:43:38.714818Z","iopub.execute_input":"2024-07-07T08:43:38.715122Z","iopub.status.idle":"2024-07-07T08:44:24.751638Z","shell.execute_reply.started":"2024-07-07T08:43:38.715097Z","shell.execute_reply":"2024-07-07T08:44:24.750699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef feature_selection(df, CFG, number_of_features):\n \"\"\"Perform feature selection using XGBoost feature importances\"\"\"\n sample_df = df.sample(frac=0.05, random_state=CFG.SEED)\n X = sample_df.drop(columns=CFG.TARGET_VARIABLE)\n y = sample_df[CFG.TARGET_VARIABLE]\n\n # Encode categorical variables\n for col in CFG.CATEGORICAL_VARIABLES:\n X[col] = LabelEncoder().fit_transform(X[col])\n model = xgb.XGBRegressor(n_estimators=100, random_state=CFG.SEED, tree_method='hist')\n\n # Fit the model\n model.fit(X, y)\n importances = model.feature_importances_\n indices = np.argsort(importances)[::-1]\n\n # Print the feature ranking\n print(\"Feature ranking:\")\n feature_ranking = [(X.columns[indices[f]], importances[indices[f]]) for f in range(X.shape[1])]\n for rank, (feature, importance) in enumerate(feature_ranking, start=1):\n print(f\"{rank}. feature {feature} ({importance})\")\n\n # Get the top features\n selected_features = X.columns[indices[:number_of_features]] # Adjust the number of features as needed\n print(\"Selected Features by XGBoost:\", selected_features)\n\n return selected_features\n\n# Assuming train_df and CFG are defined as before\nselected_features= feature_selection(train_df, CFG, 16)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:44:24.753387Z","iopub.execute_input":"2024-07-07T08:44:24.753685Z","iopub.status.idle":"2024-07-07T08:44:50.658217Z","shell.execute_reply.started":"2024-07-07T08:44:24.753659Z","shell.execute_reply":"2024-07-07T08:44:50.657494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def selected_features_dataframe(df,selected_features):\n \"\"\"Select the important features from dataframe\"\"\"\n selected_features = list(selected_features)\n selected_features.append(CFG.TARGET_VARIABLE)\n df=df[selected_features]\n return df \ntrain_df = selected_features_dataframe(train_df,selected_features)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:44:50.659766Z","iopub.execute_input":"2024-07-07T08:44:50.660327Z","iopub.status.idle":"2024-07-07T08:44:52.616012Z","shell.execute_reply.started":"2024-07-07T08:44:50.660297Z","shell.execute_reply":"2024-07-07T08:44:52.614936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nBack to top\n\n## 4 Data Pipeline \n### 4.1 Pipeline Creation ","metadata":{}},{"cell_type":"code","source":"class CustomTransformer(BaseEstimator, TransformerMixin):\n def __init__(self, func):\n self.func = func\n\n def fit(self, X, y=None):\n return self\n\n def transform(self, X, y=None):\n return self.func(X)\n\n# Define the preprocessing pipeline\ndef create_data_pipeline():\n data_pipeline = Pipeline(steps=[\n ('lagged_features_demand', CustomTransformer(lambda df: lagged_feature_creation_demand(df))),\n ('lagged_features_return', CustomTransformer(lambda df: lagged_feature_creation_return(df))),\n ('rolling_features', CustomTransformer(lambda df: rolling_feature_creation(df, 3))),\n ('memory_reduction', CustomTransformer(lambda df: reduce_memory_usage(df))),\n ('sales_growth', CustomTransformer(lambda df: sales_growth(df))),\n ('client_total_sales', CustomTransformer(lambda df: client_total_sales(df))),\n ('Final_processing', CustomTransformer(lambda df: final_processing(df,CFG.CATEGORICAL_VARIABLES))),\n ('feature_selection', CustomTransformer(lambda df: selected_features_dataframe(df,selected_features)))\n ])\n return data_pipeline\npipeline = create_data_pipeline()","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:44:52.617496Z","iopub.execute_input":"2024-07-07T08:44:52.618278Z","iopub.status.idle":"2024-07-07T08:44:52.627074Z","shell.execute_reply.started":"2024-07-07T08:44:52.618244Z","shell.execute_reply":"2024-07-07T08:44:52.626180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 4.2 Visualizing the Pipeline ","metadata":{}},{"cell_type":"code","source":"from sklearn import set_config\nset_config(display='diagram')\n\n# Display the pipeline\npipeline","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:44:52.628375Z","iopub.execute_input":"2024-07-07T08:44:52.628698Z","iopub.status.idle":"2024-07-07T08:44:52.662492Z","shell.execute_reply.started":"2024-07-07T08:44:52.628673Z","shell.execute_reply":"2024-07-07T08:44:52.661334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nBack to top\n\n## 5 Model Building \n### 5.1 Train Test Split ","metadata":{}},{"cell_type":"code","source":"def prepare_train_test_data(df, target_column='Adjusted_Demand', train_frac=8/9):\n \"\"\" Splits the data into training and testing set.\"\"\"\n X = df.drop(target_column, axis=1)\n y = df[target_column]\n # Calculate the number of unique weeks\n n_weeks = df.index.nunique()\n train_weeks = int(n_weeks * train_frac)\n # Get sorted unique index values for train and test split\n train_index = sorted(df.index.unique())[:train_weeks]\n test_index = sorted(df.index.unique())[train_weeks:]\n X_train = X.loc[train_index]\n y_train = y.loc[train_index]\n X_test = X.loc[test_index]\n y_test = y.loc[test_index]\n del X, y\n gc.collect()\n \n return X_train, y_train, X_test, y_test","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:44:52.663591Z","iopub.execute_input":"2024-07-07T08:44:52.663865Z","iopub.status.idle":"2024-07-07T08:44:52.670733Z","shell.execute_reply.started":"2024-07-07T08:44:52.663842Z","shell.execute_reply":"2024-07-07T08:44:52.669871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train, X_test, y_test = prepare_train_test_data(train_df, target_column='Adjusted_Demand', train_frac=8/9)\n# del train_df","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:44:52.671705Z","iopub.execute_input":"2024-07-07T08:44:52.671990Z","iopub.status.idle":"2024-07-07T08:47:02.882384Z","shell.execute_reply.started":"2024-07-07T08:44:52.671967Z","shell.execute_reply":"2024-07-07T08:47:02.881467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 5.2 Training the Model ","metadata":{}},{"cell_type":"code","source":"def train_and_evaluate(model_config, X_train, y_train,experiment_id, save_best_model=True, save_dir=CFG.MODEL_SAVE_PATH, categorical_features = CFG.CATEGORICAL_VARIABLES):\n \"\"\"\n Train and evaluate a model using time series cross-validation.\n \"\"\"\n tscv = TimeSeriesSplit(n_splits=2)\n cv_scores = []\n best_score = float('inf')\n best_model = None\n model_type = model_config['model'].__name__ \n model = model_config['model'](**model_config['params'])\n with mlflow.start_run(experiment_id=experiment_id):\n mlflow.log_param(\"model_name\", model_type)\n for train_idx, val_idx in tscv.split(X_train):\n X_train_fold, X_val_fold = X_train.iloc[train_idx], X_train.iloc[val_idx]\n y_train_fold, y_val_fold = y_train.iloc[train_idx], y_train.iloc[val_idx]\n # Train the model\n fold_model = clone(model) # Clone the model for independent training\n # Train the model\n if model_type == 'LGBMRegressor':\n model.fit(X_train_fold, y_train_fold, categorical_feature=categorical_features)\n elif model_type == 'XGBRegressor':\n model.fit(X_train_fold, y_train_fold, eval_set=[(X_val_fold, y_val_fold)], early_stopping_rounds=10, verbose=False)\n elif model_type == 'CatBoostRegressor':\n model.fit(X_train_fold, y_train_fold, eval_set=[(X_val_fold, y_val_fold)], cat_features=categorical_features, early_stopping_rounds=10, verbose=False)\n else:\n model.fit(X_train_fold, y_train_fold)\n\n # Evaluate the model\n y_val_pred = model.predict(X_val_fold)\n val_rmse = np.sqrt(mean_squared_error(y_val_fold, y_val_pred))\n cv_scores.append(val_rmse)\n mlflow.log_metric(f\"val_rmse_fold_{len(cv_scores)}\", val_rmse)\n\n # Check if the current fold's model is the best one for this algorithm\n if val_rmse < best_score:\n best_score = val_rmse\n best_model = model\n\n mlflow.log_params(model_config['params'])\n mlflow.log_metric(\"best_val_rmse\", best_score)\n # Save the best model for this algorithm\n best_model_filename = f\"{model_type}_best_model.joblib\"\n best_model_save = os.path.join(CFG.MODEL_SAVE_PATH, experiment_id)\n check_and_create_directory(best_model_save)\n joblib.dump(best_model, os.path.join(best_model_save,best_model_filename))\n print(f\"Best model ({model_type}) with RMSE {best_score:.4f} saved to {best_model_filename}\")\n\n return np.mean(cv_scores)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:47:02.883625Z","iopub.execute_input":"2024-07-07T08:47:02.883925Z","iopub.status.idle":"2024-07-07T08:47:02.896846Z","shell.execute_reply.started":"2024-07-07T08:47:02.883900Z","shell.execute_reply":"2024-07-07T08:47:02.895762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initiating for Experiment tracking\ncheck_and_create_directory(CFG.MODEL_SAVE_PATH) # to create directory to save models\ncheck_and_create_directory(CFG.MLFLOW_LOCAL_URI) # to create directory for experiment tracking\nif CFG.EXPERIMENT_RECORDING =='LOCAL':\n mlflow.set_tracking_uri(CFG.MLFLOW_LOCAL_URI)\nelse:\n dagshub.init(CFG.DAGSHUB_REPO, CFG.DAGSHUB_REPO_OWNER, mlflow=True)\nexperiment_id = get_or_create_experiment_id(CFG.MLFLOW_EXPERIMENT_NAME)\nfor model_name, model_config in CFG.MODELS.items():\n print(f\"Training and evaluating model: {model_name}\")\n average_rmse = train_and_evaluate(model_config, X_train, y_train,experiment_id)\n print(f\"Average RMSE for {model_name}: {average_rmse:.3f}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-07T08:47:02.898175Z","iopub.execute_input":"2024-07-07T08:47:02.898592Z","iopub.status.idle":"2024-07-07T09:11:12.944012Z","shell.execute_reply.started":"2024-07-07T08:47:02.898558Z","shell.execute_reply":"2024-07-07T09:11:12.943017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 5.3 Performance Evaluation ","metadata":{}},{"cell_type":"code","source":"check = pd.DataFrame()\ndef load_and_evaluate_models(model_directory, X_test, y_test,experiment_id):\n \"\"\"Evaluation the Performance of Various Model using Test Data\"\"\"\n # List all files in the model directory\n print(os.listdir(os.path.join(model_directory,experiment_id)))\n model_files = [f for f in os.listdir(os.path.join(model_directory,experiment_id)) if f.endswith('.joblib')]\n # Iterate over the found model files\n for model_file in model_files:\n # Load the model\n model_path = os.path.join(model_directory, experiment_id, model_file)\n model = joblib.load(model_path)\n print(f\"Loaded model from {model_path}\")\n\n # Predict on the test set\n y_test_pred = model.predict(X_test)\n y_test = y_test.astype(float)\n y_test_pred = y_test_pred.astype(float)\n # Evaluate the model on the test set\n test_mse = mean_squared_error(y_test, y_test_pred)\n test_rmse = np.sqrt(test_mse)\n test_r2 = r2_score(y_test, y_test_pred)\n print(f'Test MSE: {test_mse}')\n print(f'Test RMSE: {test_rmse}')\n print(f'Test R-squared: {test_r2}')\n # Plot actual vs predicted values for test set\n plt.figure(figsize=(10, 6))\n plt.plot(y_test.values, label='Actual')\n plt.plot(y_test_pred, label='Predicted')\n plt.title(f'Actual vs Predicted Adjusted Demand (Test Set) - {model_file}')\n plt.xlabel('Samples')\n plt.ylabel('Adjusted Demand')\n plt.legend()\n plt.show()\n del model\n# del y_test_pred\nmodel_directory = CFG.MODEL_SAVE_PATH\nload_and_evaluate_models(model_directory, X_test, y_test,experiment_id)\n# Cleanup after execution\ndel X_train\ndel y_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:11:12.945418Z","iopub.execute_input":"2024-07-07T09:11:12.946085Z","iopub.status.idle":"2024-07-07T09:13:36.846801Z","shell.execute_reply.started":"2024-07-07T09:11:12.946051Z","shell.execute_reply":"2024-07-07T09:13:36.845783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 5.4 Ensembling for Performance Improvement ","metadata":{}},{"cell_type":"code","source":"def load_models(model_directory):\n \"\"\"Load all models from the specified directory.\"\"\"\n models = {}\n for model_name in ['XGBRegressor_best', 'CatBoostRegressor_best']:\n file_path = os.path.join(model_directory, f\"{model_name}_model.joblib\")\n models[model_name] = joblib.load(file_path)\n return models\n\n# Directory where models are saved\nmodel_directory = '/kaggle/working/models/' + experiment_id\n\n# Load models\nmodels = load_models(model_directory)\n\ndef objective(trial):\n \"\"\"Enhance the performance of ensembling and optimizing weights.\"\"\"\n # Suggest weights for each model\n w_catboost = trial.suggest_float('w_catboost', 0, 1)\n w_xgboost = trial.suggest_float('w_xgboost', 0, 1)\n \n # Normalize weights so they sum to 1\n total = w_catboost + w_xgboost\n w_catboost /= total\n w_xgboost /= total\n \n # Collect individual model predictions\n y_pred_cat = models['CatBoostRegressor_best'].predict(X_test)\n y_pred_xgb = models['XGBRegressor_best'].predict(X_test)\n\n # Combine predictions based on the weights\n y_pred = (w_catboost * y_pred_cat + w_xgboost * y_pred_xgb)\n\n # Calculate and return the RMSE\n rmse = np.sqrt(mean_squared_error(y_test, y_pred))\n return rmse\n\n# Set up Optuna study\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=10)\n\n# Retrieve the best weights\nbest_weights = {\n 'CatBoostRegressor_best': study.best_trial.params['w_catboost'],\n 'XGBRegressor_best': study.best_trial.params['w_xgboost']\n}\ntotal = sum(best_weights.values())\nbest_weights = {model: weight / total for model, weight in best_weights.items()}\nprint(\"Best weights:\", best_weights)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:13:36.847965Z","iopub.execute_input":"2024-07-07T09:13:36.848266Z","iopub.status.idle":"2024-07-07T09:23:06.187682Z","shell.execute_reply.started":"2024-07-07T09:13:36.848240Z","shell.execute_reply":"2024-07-07T09:23:06.186770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_ensemble(X_test, model_directory, best_weights):\n \"\"\"Make predictions using an ensemble of pre-trained models and specified weights.\"\"\"\n models = load_models(model_directory)\n # Collect individual model predictions\n y_pred_cat = models['CatBoostRegressor_best'].predict(X_test)\n y_pred_lgb = models['XGBRegressor_best'].predict(X_test)\n\n # Combine predictions based on the weights\n y_pred = (best_weights['CatBoostRegressor_best'] * y_pred_cat +\n best_weights['XGBRegressor_best'] * y_pred_lgb)\n return y_pred","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:23:06.188827Z","iopub.execute_input":"2024-07-07T09:23:06.189135Z","iopub.status.idle":"2024-07-07T09:23:06.194609Z","shell.execute_reply.started":"2024-07-07T09:23:06.189109Z","shell.execute_reply":"2024-07-07T09:23:06.193636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 5.5 Model Explainablity ","metadata":{}},{"cell_type":"code","source":"def model_explainability(model_directory, X_test):\n \"\"\"Provide the importance of Features in the Model\"\"\"\n shap_values_dict = {}\n X_test_sampled = X_test.sample(frac=0.005) # Ensure variable naming consistency\n models = load_models(model_directory)\n for name, model in models.items():\n print(f\"Explaining model: {name}\")\n if 'catboost' in name.lower():\n explainer = shap.TreeExplainer(model, feature_perturbation='interventional', check_additivity=False)\n shap_values = explainer(X_test_sampled, check_additivity=False)\n elif 'xgb' in name.lower() or 'lgbm' in name.lower():\n explainer = shap.TreeExplainer(model, check_additivity=False)\n shap_values = explainer(X_test_sampled, check_additivity=False)\n else:\n explainer = shap.Explainer(model, feature_perturbation='interventional', check_additivity=False)\n shap_values = explainer(X_test_sampled)\n \n shap_values_dict[name] = shap_values\n \n # Plot summary plot for the current model\n shap.summary_plot(shap_values, X_test_sampled, show=False)\n plt.title(f\"SHAP Summary Plot for {name}\")\n plt.show()\n \n # Plot feature importance bar chart for the current model\n shap.plots.bar(shap_values, show=False)\n plt.title(f\"SHAP Feature Importance for {name}\")\n plt.show()\n \n return shap_values_dict\n\n# Example usage\nshap_values_dict = model_explainability(model_directory, X_test)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:23:06.195671Z","iopub.execute_input":"2024-07-07T09:23:06.195938Z","iopub.status.idle":"2024-07-07T09:24:10.853514Z","shell.execute_reply.started":"2024-07-07T09:23:06.195916Z","shell.execute_reply":"2024-07-07T09:24:10.852505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nBack to top\n\n## 6 Model Inference \n\n### 6.1 Loading the Model ","metadata":{}},{"cell_type":"code","source":"def predict_ensemble(X_test, model_directory, best_weights):\n \"\"\"Make predictions using an ensemble of pre-trained models and specified weights.\"\"\"\n models = load_models(model_directory)\n # Collect individual model predictions\n y_pred_cat = models['CatBoostRegressor_best'].predict(X_test)\n y_pred_lgb = models['XGBRegressor_best'].predict(X_test)\n\n # Combine predictions based on the weights\n y_pred = (best_weights['CatBoostRegressor_best'] * y_pred_cat +\n best_weights['XGBRegressor_best'] * y_pred_lgb)\n return y_pred","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:24:10.858162Z","iopub.execute_input":"2024-07-07T09:24:10.858453Z","iopub.status.idle":"2024-07-07T09:24:10.864055Z","shell.execute_reply.started":"2024-07-07T09:24:10.858429Z","shell.execute_reply":"2024-07-07T09:24:10.863153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 6.2 Data Loading and Prepration \n#### 6.2.1 Reading the data","metadata":{}},{"cell_type":"code","source":"def read_filtered_csv(file_path, Week, chunksize=10000000):\n \"\"\" Reading the data in chunks\"\"\"\n filtered_df = pd.DataFrame()\n filter_condition = lambda df: df['Semana'] >= Week-2\n for chunk in pd.read_csv(file_path, compression='zip', chunksize=chunksize):\n # Apply the filter condition\n filtered_chunk = chunk[filter_condition(chunk)]\n # Append the filtered chunk to the final DataFrame\n filtered_df = pd.concat([filtered_df, filtered_chunk], ignore_index=True)\n return filtered_df\ntest_df = read_filtered_csv(CFG.TRAIN_PATH,9 , chunksize=10000000)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:24:10.865325Z","iopub.execute_input":"2024-07-07T09:24:10.865771Z","iopub.status.idle":"2024-07-07T09:25:30.520700Z","shell.execute_reply.started":"2024-07-07T09:24:10.865737Z","shell.execute_reply":"2024-07-07T09:25:30.519673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 6.2.2 Processing the Data","metadata":{}},{"cell_type":"code","source":"def processing_for_prediction(df,week):\n df = prepare_data(df)\n df = pipeline.fit_transform(df)\n week_for_prediction = pd.to_datetime('2024-01-01') + pd.to_timedelta(week * 7, unit='D')\n df = df[df.index == week_for_prediction]\n df = df.drop('Adjusted_Demand',axis = 1)\n return df","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:25:30.522082Z","iopub.execute_input":"2024-07-07T09:25:30.522389Z","iopub.status.idle":"2024-07-07T09:25:30.528053Z","shell.execute_reply.started":"2024-07-07T09:25:30.522363Z","shell.execute_reply":"2024-07-07T09:25:30.526950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = processing_for_prediction(test_df,9)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:25:30.529294Z","iopub.execute_input":"2024-07-07T09:25:30.529655Z","iopub.status.idle":"2024-07-07T09:28:20.891383Z","shell.execute_reply.started":"2024-07-07T09:25:30.529630Z","shell.execute_reply":"2024-07-07T09:28:20.890495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 6.3 Model Forecasting ","metadata":{}},{"cell_type":"code","source":"def prediction(df):\n y_pred = predict_ensemble(df, model_directory, best_weights)\n df['Forecasted Demand'] = y_pred\n return df","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:28:20.892661Z","iopub.execute_input":"2024-07-07T09:28:20.893021Z","iopub.status.idle":"2024-07-07T09:28:20.898150Z","shell.execute_reply.started":"2024-07-07T09:28:20.892989Z","shell.execute_reply":"2024-07-07T09:28:20.897316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = prediction(test_df)\ntest_df.head(1000).to_csv('Forecasted_demand.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:28:20.899776Z","iopub.execute_input":"2024-07-07T09:28:20.900121Z","iopub.status.idle":"2024-07-07T09:29:54.912248Z","shell.execute_reply.started":"2024-07-07T09:28:20.900091Z","shell.execute_reply":"2024-07-07T09:29:54.911412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Mlflow Local UI","metadata":{}},{"cell_type":"code","source":"if CFG.EXPERIMENT_RECORDING == 'LOCAL':\n !pip install mlflow --quiet\n !pip install pyngrok --quiet\n \n from pyngrok import ngrok\n from getpass import getpass\n\n ngrok.kill()\n\n # Setting the authtoken (replace with your own token from https://dashboard.ngrok.com/auth)\n NGROK_AUTH_TOKEN = \"2ioyKWLGBeFWH5D7vI5ktAJaZSV_2aUaXGdGJeHsk43qWXRvk\"\n ngrok.set_auth_token(NGROK_AUTH_TOKEN)\n\n # Start the MLflow UI\n get_ipython().system_raw(\"mlflow ui --backend-store-uri file:/kaggle/working/mlflow --host 0.0.0.0 --port 5000 &\")\n\n # Open an HTTPs tunnel on port 5000 for http://localhost:5000\n ngrok_tunnel = ngrok.connect(addr=\"5000\", proto=\"http\", bind_tls=True)\n print(\"MLflow Tracking UI:\", ngrok_tunnel.public_url)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-07T09:29:54.913500Z","iopub.execute_input":"2024-07-07T09:29:54.913940Z","iopub.status.idle":"2024-07-07T09:30:24.086094Z","shell.execute_reply.started":"2024-07-07T09:29:54.913907Z","shell.execute_reply":"2024-07-07T09:30:24.084951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}