{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"none","dataSources":[{"sourceId":103436,"databundleVersionId":12476472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install catboost","metadata":{"id":"Wsakg6FVy0Ki","outputId":"530f0f1c-f72a-4a1b-a53a-dc7e07614b88"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install imbalanced-learn optuna","metadata":{"id":"frlevojPzMIH","outputId":"47417a63-520d-4c8c-a446-86ea114218f9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from google.colab import drive\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nimport json\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier, Pool\nfrom lightgbm import LGBMClassifier\nfrom sklearn.metrics import classification_report, roc_auc_score, f1_score, precision_score, recall_score\nfrom imblearn.under_sampling import RandomUnderSampler\n\nimport shap\n\nimport optuna","metadata":{"id":"vtkWyivXy1bL"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read data","metadata":{"id":"pFD9znXBy4ps"}},{"cell_type":"code","source":"drive.mount('/content/drive')","metadata":{"id":"_n2fQQe7y3MJ","outputId":"da13d682-9cd9-4a7c-dc15-e7406b294dd4"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_delay_4_6 = pd.read_csv('/content/drive/MyDrive/DS108/Datasets/Copy of delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv', encoding='utf-8')\ndf_not_delay_4_6 = pd.read_csv('/content/drive/MyDrive/DS108/Datasets/Copy of not_delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv', encoding='utf-8')\n\ndf_delay_7_9 = pd.read_csv('/content/drive/MyDrive/DS108/Datasets/Copy of delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv', encoding='utf-8')\ndf_not_delay_7_9 = pd.read_csv('/content/drive/MyDrive/DS108/Datasets/Copy of not_delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv', encoding='utf-8')","metadata":{"id":"UCUbhWz4y66C","outputId":"5095d591-ea87-4182-dd87-ee18252328ba"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_4_6 = pd.concat([df_delay_4_6, df_not_delay_4_6], axis=0, ignore_index=True)\ndf_4_6.drop_duplicates(inplace=True)\n\ndf_7_9 = pd.concat([df_delay_7_9, df_not_delay_7_9], axis=0, ignore_index=True)\ndf_7_9.drop_duplicates(inplace=True)\n\ncommon_cols = df_4_6.columns.intersection(df_7_9.columns)\ndf_full = pd.concat([df_4_6[common_cols], df_7_9[common_cols]], axis=0, ignore_index=True)\ndf_full.drop_duplicates(inplace=True)","metadata":{"id":"LKKnQMbXy8Qq"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_10 = pd.read_csv('/content/drive/MyDrive/DS108/Datasets/PILOT_10.csv', encoding='utf-8')","metadata":{"id":"oil1mcDXboHn"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Shape of df_4_6:', df_4_6.shape)\nprint('Shape of df_7_9:', df_7_9.shape)\nprint('Shape of df_full:', df_full.shape)\nprint('Shape of df_10:', df_10.shape)","metadata":{"id":"9sxVfZaHy9l6","outputId":"f31c6b09-6555-46d6-db9a-67ba0b05f663"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_10.info()","metadata":{"id":"AwpytOQhbmaU","outputId":"7810096e-c1e2-4143-cf6f-820a7373cc68"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_full['Order date'] = pd.to_datetime(df_full['Order date'], format='mixed')","metadata":{"id":"Ak9xDNngZAJn"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Practice","metadata":{"id":"3G2ppJw-W_V1"}},{"cell_type":"code","source":"def missing_value_report(df, sort=True, ascending=False):\n    total_missing = df.isnull().sum()\n    percent_missing = 100 * total_missing / len(df)\n    missing_df = pd.DataFrame({\n        'Missing Count': total_missing,\n        'Missing Rate (%)': percent_missing\n    })\n    missing_df = missing_df[missing_df['Missing Count'] > 0]\n\n    if sort:\n        missing_df = missing_df.sort_values(by='Missing Rate (%)', ascending=ascending)\n\n    return missing_df","metadata":{"id":"nQZImSiBcD8P"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_value_report(df_10)","metadata":{"id":"1r3r-Xmue2tP","outputId":"fa05e9af-92ba-4b41-9bf6-f61aca3e380b"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clean 'OTHER AREA SHIP DIV' column\ndf_full['OTHER AREA SHIP DIV'] = df_full['OTHER AREA SHIP DIV'].fillna(0)\ndf_full['OTHER AREA SHIP DIV'] = df_full['OTHER AREA SHIP DIV'].replace([' '], 0)\ndf_full['OTHER AREA SHIP DIV'] = df_full['OTHER AREA SHIP DIV'].astype('int64')","metadata":{"id":"E0njXmrCXIBu"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# extract hour, minute, second as integers\npadded = df_full['SO_TIME'].astype(str).str.zfill(6)\ndf_full['hour order'] = padded.str.slice(0, 2).astype(int)\ndf_full['minute order'] = padded.str.slice(2, 4).astype(int)\ndf_full['second order'] = padded.str.slice(4, 6).astype(int)\n\n# convert to datetime, then extract month and day\ndate_cols = ['VSD']\nfor col in date_cols:\n  df_full[col] = pd.to_datetime(df_full[col], format='mixed')\n  df_full[col + ' month'] = df_full[col].dt.month\n  df_full[col + ' day'] = df_full[col].dt.day","metadata":{"id":"fG8VgUr4a2fu"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tạo cột day_range bằng cách trừ hai cột datetime\ndf_full['day_range'] = (df_full['VSD'] - df_full['Order date']).dt.days","metadata":{"id":"Q3o5ClMVclHT"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_full['SO_DAY_OF_WEEK_sin'] = np.sin(2 * np.pi * df_full['SO_DAY_OF_WEEK'] / 7)\ndf_full['SO_DAY_OF_WEEK_cos'] = np.cos(2 * np.pi * df_full['SO_DAY_OF_WEEK'] / 7)\n# Tương tự cho hour order (chia cho 24) và SO_DAY_OF_MONTH (chia cho số ngày max của tháng đó, thường là 31)\ndf_full['SO_DAY_OF_MONTH_sin'] = np.sin(2 * np.pi * df_full['SO_DAY_OF_MONTH'] / 31)\ndf_full['SO_DAY_OF_MONTH_cos'] = np.cos(2 * np.pi * df_full['SO_DAY_OF_MONTH'] / 31)\n\ndf_full['hour_order_sin'] = np.sin(2 * np.pi * df_full['hour order'] / 24)\ndf_full['hour_order_cos'] = np.cos(2 * np.pi * df_full['hour order'] / 24)","metadata":{"id":"bt-G4or7iFF1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_full['Is_Weekend'] = df_full['Order date'].dt.dayofweek.isin([5, 6]).astype(int)","metadata":{"id":"1JkEf2iGi2c1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_not_9['Allocation_Ratio'] = df_not_9['ALLOCATION QTY'] / df_not_9['SO QTY'] # Tỷ lệ phân bổ\ndf_full['Total_Weight'] = df_full['PACK QTY'] * df_full['WEIGHT PER PIECE'] # Tổng trọng lượng gói hàng\ndf_full['Weight'] = df_full['SO QTY'] * df_full['WEIGHT PER PIECE'] # Tổng trọng lượng hàng","metadata":{"id":"m_kI6RYjjHVh"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_full['Consider count hodiday Saturday'] = df_full['Consider count hodiday Saturday'].replace(' ', 0).astype(int)","metadata":{"id":"nhHCY8CKk1mH"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Columns of full data:', df_full.columns)","metadata":{"id":"KJUj5Q_EcT58","outputId":"b011000d-e9d2-4f83-809c-b710b0e0d592"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_value_report(df_full)","metadata":{"id":"xXFD4wH-cZx0","outputId":"16f89d6a-40a1-4bd1-fe50-cb5342a5d41a"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_full.info()","metadata":{"id":"PmAGzbQNeH6c","outputId":"a4e53f73-c619-492e-dc91-303145f2c631"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_drop = ['QTUF_RCV_NO', 'SOUF_RCV_NO', 'REASON_CD', 'SHIP DECISION NO', 'second order',\n             'GLOBAL_NO', 'VSD month', 'minute order', 'SO_TIME', 'SPECIAL_DIV']\n\nrows_drop = ['Ship Mode', 'SUPPLIER_DIV']\n\ndf_full = df_full.drop(columns=cols_drop)\ndf_full = df_full.dropna(subset=rows_drop)","metadata":{"id":"0eieebB-eMVx"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## df 9","metadata":{"id":"W3PZwnfbslpk"}},{"cell_type":"code","source":"df_9 = df_full[df_full['Order date'].dt.month == 9]\ndf_not_9 = df_full[df_full['Order date'].dt.month != 9]","metadata":{"id":"_5GiyChUslTi"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_9 = df_9.drop(columns=['Order date', 'VSD'])\ndf_not_9 = df_not_9.drop(columns=['Order date', 'VSD'])","metadata":{"id":"KLARFxR7krZR"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Correlation","metadata":{"id":"TC8MsWyXp-yV"}},{"cell_type":"code","source":"correlation_matrix = df_not_9.corr(numeric_only=True)","metadata":{"id":"uZRse_pbpYb1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_with_label = correlation_matrix['label'].sort_values(ascending=False)\nprint(\"\\nMối tương quan với biến 'label':\")\nprint(correlation_with_label)","metadata":{"id":"3DLxqMDVqAJU","outputId":"48eee9d0-5a73-42df-89f4-8265387cba9c"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 10))\nsns.heatmap(correlation_matrix, annot=False, cmap='coolwarm', fmt=\".2f\", linewidths=.5)\nplt.title('Ma trận tương quan giữa các thuộc tính')\nplt.show()\n\n# Hoặc chỉ trực quan mối tương quan với 'label'\nplt.figure(figsize=(8, 6))\nsns.barplot(x=correlation_with_label.index, y=correlation_with_label.values, palette='viridis')\nplt.title('Mối tương quan của các thuộc tính với Label')\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"id":"Fe0ancuUqEns","outputId":"c5fe7477-38ab-464f-89f6-914f5f95955b"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocess","metadata":{"id":"oTt-r2urt9yf"}},{"cell_type":"code","source":"# def clip_outliers(train_df, apply_dfs, columns=None, lower_q=0.01, upper_q=0.99):\n#     if columns is None:\n#         columns = train_df.select_dtypes(include=['number']).columns.tolist()\n\n#     # Compute clipping thresholds from train data\n#     lower_bounds = train_df[columns].quantile(lower_q)\n#     upper_bounds = train_df[columns].quantile(upper_q)\n\n#     # Clip train\n#     for col in columns:\n#         train_df[col] = train_df[col].clip(lower_bounds[col], upper_bounds[col])\n\n#     # Clip other data\n#     for df in apply_dfs:\n#         for col in columns:\n#             df[col] = df[col].clip(lower_bounds[col], upper_bounds[col])","metadata":{"id":"xyNWf-vbqpun"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## WOE & IV","metadata":{"id":"4W2VBhGjuR2J"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef iv_woe(data, target, bins=10, show_woe=False, force_categorical=[]):\n\n    newDF, woeDF = pd.DataFrame(), pd.DataFrame()\n    cols = data.columns\n\n    for ivars in cols[~cols.isin([target])]:\n        temp_data = data[[ivars, target]].dropna()\n\n        # convert to categorical data\n        if ivars in force_categorical:\n            d0 = pd.DataFrame({'x': temp_data[ivars].astype(str), 'y': temp_data[target]})\n\n        # if numeric plit bin qcut\n        elif (temp_data[ivars].dtype.kind in 'bifc') and (len(np.unique(temp_data[ivars])) > 10):\n            try:\n                binned_x = pd.qcut(temp_data[ivars], bins, duplicates='drop')\n                d0 = pd.DataFrame({'x': binned_x, 'y': temp_data[target]})\n            except:\n                print(f\"Không thể bin cột {ivars}, có thể do giá trị trùng nhau quá nhiều.\")\n                continue\n\n        #\n        else:\n            d0 = pd.DataFrame({'x': temp_data[ivars].astype(str), 'y': temp_data[target]})\n\n        d = d0.groupby(\"x\", as_index=False).agg({\"y\": [\"count\", \"sum\"]})\n        d.columns = ['Cutoff', 'N', 'Events']\n        d['% of Events'] = np.maximum(d['Events'], 0.5) / d['Events'].sum()\n        d['Non-Events'] = d['N'] - d['Events']\n        d['% of Non-Events'] = np.maximum(d['Non-Events'], 0.5) / d['Non-Events'].sum()\n        d['WoE'] = np.log(d['% of Events']/d['% of Non-Events'])\n        d['IV'] = d['WoE'] * (d['% of Events'] - d['% of Non-Events'])\n        d.insert(loc=0, column='Variable', value=ivars)\n        print(\"Information value of \" + ivars + \" is \" + str(round(d['IV'].sum(),6)))\n        temp =pd.DataFrame({\"Variable\" : [ivars], \"IV\" : [d['IV'].sum()]}, columns = [\"Variable\", \"IV\"])\n        newDF=pd.concat([newDF,temp], axis=0)\n        woeDF=pd.concat([woeDF,d], axis=0)\n\n        if show_woe == True:\n            print(d)\n\n    return newDF.sort_values(by=\"IV\", ascending=False).reset_index(drop=True), woeDF","metadata":{"id":"0M2ORzP0uBL0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_cols = ['CLASSIFY_CD', 'CUST_CD','BRAND_CD', 'INNER_CD', 'SUPPLIER_CD','Stock class',\n                    'OTHER AREA SHIP DIV', 'PACKING RANK', 'PRODUCT_CD', 'PRODUCT ATTRIBUTION',\n                    'SPECIAL DIV','LOGICAL PLANT', 'DIRECT SHIP FLG','DELI_DIV', 'Ship Mode',\n                    'SHIP DECISION NO', 'SUPPLIER_DIV', 'SPECIAL_DIV', 'Is_Weekend', ]\n\niv, woe = iv_woe(\n    data=df_not_9,\n    target='label',\n    bins=15,\n    force_categorical=categorical_cols\n)\n\nprint(iv)","metadata":{"id":"1pfctaHXuVjY","outputId":"83aff2bd-c7aa-4570-b5d1-fc8f453d4431"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"use_cols = ['PRODUCT_CD', 'INNER_CD', 'CUST_CD', 'CLASSIFY_CD', 'DELI_DIV', 'SUPPLIER_CD', 'DIRECT SHIP FLG',\n            'BRAND_CD', 'day_range', 'Weight', 'WEIGHT PER PIECE', 'SUPPLIER INV AMOUNT', 'PURCHASE AMOUNT',\n            'VSD day', 'Ship Mode']","metadata":{"id":"0awtFpQa7kAz"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train Dev","metadata":{"id":"o8SC3kNF27J2"}},{"cell_type":"code","source":"X_train = df_not_9.drop(columns=['label']) # Assuming 'label' is the target column\ny_train = df_not_9['label']\nX_test = df_9.drop(columns=['label'])\ny_test = df_9['label']","metadata":{"id":"6XTCEFBb23Bj"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model","metadata":{"id":"KV9h_o-F1rF8"}},{"cell_type":"markdown","source":"### Catboost","metadata":{"id":"_iqzxLJFjg6K"}},{"cell_type":"code","source":"df_not_9.info()","metadata":{"id":"aGnYv2KVjl23","outputId":"0f39e4df-7d15-4508-d380-80cba5f92169"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = ['SUBSIDIARY_CD', 'CLASSIFY_CD', 'CUST_CD', 'BRAND_CD', 'INNER_CD', 'SUPPLIER_CD',\n                        'Sales order line number', 'Stock class', 'OTHER AREA SHIP DIV', 'PRODUCT_CD',\n                        'PRODUCT ATTRIBUTION', 'SPECIAL DIV', 'LOGICAL PLANT', 'DIRECT SHIP FLG',\n                        'DELI_DIV', 'Ship Mode', 'SUPPLIER_DIV', 'Is_Weekend', 'PACKING RANK']","metadata":{"id":"bouPLGgI15NL"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in categorical_features:\n    X_train[col] = X_train[col].astype(str)\n    X_test[col] = X_test[col].astype(str)","metadata":{"id":"4CZ-f4X-qq6w"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import Pool, CatBoostClassifier\n\n\ntrain_pool = Pool(\n    data=X_train,\n    label=y_train,\n    cat_features=categorical_features\n)\n\ncatboost_model = CatBoostClassifier(\n    iterations=200,\n    learning_rate=0.15,\n    depth=6,\n    random_seed=42,\n    auto_class_weights='Balanced',\n    loss_function='Logloss',\n    verbose=0 # Tắt log để gọn\n)\n\ncatboost_model.fit(train_pool)","metadata":{"id":"8EFyTj8jm34c","outputId":"5871de5c-1292-4a52-f47e-d5b4ac2b5733"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"explainer = shap.TreeExplainer(catboost_model)\nshap_values = explainer.shap_values(X_test)","metadata":{"id":"9CqkOBFI0TEE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vẽ biểu đồ tóm tắt SHAP\nshap.summary_plot(shap_values, X_test, max_display=X_test.shape[1])","metadata":{"id":"YsZp4I0z0yJB"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Chuyển đổi giá trị SHAP thành DataFrame\nshap_values_df = pd.DataFrame(shap_values, columns=X_test.columns)\n\n# Xem các thống kê mô tả của các giá trị SHAP\nshap_values_df.describe().T","metadata":{"id":"g0z4Lhfe03Xj","outputId":"c35d1fd1-378f-4baf-8e6b-d0f37cc58db5"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_cat = catboost_model.predict(X_test)\n\n# F1-score\nf1_CB = f1_score(y_test, y_pred_cat)\nprecision_CB = precision_score(y_test, y_pred_cat)\nrecall_CB = recall_score(y_test, y_pred_cat)\n\nprint(\"F1-score (test):\", round(f1_CB, 4))\nprint(\"Precision (test):\", round(precision_CB, 4))\nprint(\"Recall (test):\", round(recall_CB, 4))\n\n# report\nprint(\"\\nClassification Report:\\n\", classification_report(y_test, y_pred_cat, digits=4))","metadata":{"id":"0QuoKf-ypoU1","outputId":"5a6657cc-beab-4cd7-ffc0-274fc247a364"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### UnderSampling","metadata":{"id":"WytuNfiPwW5i"}},{"cell_type":"code","source":"# Import các thư viện cần thiết\nimport pandas as pd\nfrom imblearn.over_sampling import SMOTE\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom collections import Counter","metadata":{"id":"uUdXaNaiw-nS"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Trước khi undersample và oversample: {Counter(y_train)}\")\n\n# Calculate target counts based on desired ratios (majority:minority)\noriginal_minority_count = Counter(y_train)[1]\ndesired_majority_under = 17 * original_minority_count\ndesired_minority_under = original_minority_count\n\nunder_sampler = RandomUnderSampler(sampling_strategy={0: desired_majority_under, 1: desired_minority_under}, random_state=42)\nX_under, y_under = under_sampler.fit_resample(X_train, y_train)\nprint(f\"Sau khi under-sample (30:1 majority:minority): {Counter(y_under)}\")","metadata":{"id":"ke0PLqmBrwPj","outputId":"559ab4ba-5a19-4ce8-8741-fc7ce54131fe"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import Pool, CatBoostClassifier\n\n\ntrain_pool = Pool(\n    data=X_under,\n    label=y_under,\n    cat_features=categorical_features\n)\n\ncatboost_model = CatBoostClassifier(\n    iterations=191,\n    learning_rate=0.23167839441737345,\n    depth=5,\n    l2_leaf_reg= 4.090354306055502,\n    border_count= 51,\n    random_seed=42,\n    loss_function='Logloss',\n    eval_metric= \"F1\",\n    verbose=0, # Tắt log để gọn\n    task_type= \"GPU\"\n)\n\ncatboost_model.fit(train_pool)","metadata":{"id":"1u-ocZiEk_Ky","outputId":"def5f74f-514b-4873-808e-0c0fff2997eb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = catboost_model.predict(X_test)\nf1 = f1_score(y_test, y_pred)\nf1","metadata":{"id":"uBqaJf1HoEuK","outputId":"cfb11586-23d6-4bf8-a817-e956738605bc"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"explainer = shap.TreeExplainer(catboost_model)\nshap_values = explainer.shap_values(X_test)","metadata":{"id":"fyy44arklYS3"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# day_range, SUPPLIER_CD, CUST_CD, VSD day, BRAND_CD, CLASSIFY_CD, DELI_DIV, PURCHASE AMOUNT, PRODUCT_CD, INNER_CD, SUPPLIER INV AMOUNT","metadata":{"id":"OJysgJmbvnSE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 0                        PRODUCT_CD  35.591908\n# 1                          INNER_CD  10.325964\n# 2                           CUST_CD   3.207646\n# 3                       CLASSIFY_CD   1.651734\n# 4                          DELI_DIV   1.585817\n# 5                       SUPPLIER_CD   1.580959\n# 6                   DIRECT SHIP FLG   0.709505\n# 7                          BRAND_CD   0.333186\n# 8                         day_range   0.324085\n# 9                            Weight   0.302927\n# 10                 WEIGHT PER PIECE   0.243452\n# 11              SUPPLIER INV AMOUNT   0.221067\n# 12                  PURCHASE AMOUNT   0.220906\n# 13                          VSD day   0.173217\n# 14                        Ship Mode   0.155638\n# 15                     SUPPLIER_DIV   0.069567\n# 16              OTHER AREA SHIP DIV   0.069318\n# 17                   ALLOCATION QTY   0.050487\n# 18                           SO QTY   0.050487\n# 19                  SO_DAY_OF_MONTH   0.039946\n# 20          Sales order line number   0.035888\n# 21              SO_DAY_OF_MONTH_cos   0.035799\n# 22  Consider count hodiday Saturday   0.033336\n# 23              SO_DAY_OF_MONTH_sin   0.021692\n# 24                    LOGICAL PLANT   0.021422\n# 25                   hour_order_sin   0.017975\n# 26                   hour_order_cos   0.017131\n# 27                       hour order   0.014757\n# 28                      SPECIAL DIV   0.013600\n# 29               SO_DAY_OF_WEEK_sin   0.008007\n# 30                   SO_DAY_OF_WEEK   0.008007\n# 31               SO_DAY_OF_WEEK_cos   0.008007\n# 32                     PACKING RANK   0.003690\n# 33              PRODUCT ATTRIBUTION   0.002568\n# 34                      Stock class   0.002568\n# 35                       Is_Weekend   0.001024\n# 36                    SUBSIDIARY_CD   0.000000\n# 37                         PACK QTY   0.000000\n# 38                     Total_Weight   0.000000","metadata":{"id":"v6Jk0NpYGfcr"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vẽ biểu đồ tóm tắt SHAP\nshap.summary_plot(shap_values, X_test, max_display=X_test.shape[1])","metadata":{"id":"ltF7Ua2Klakg","outputId":"ccb94121-7d4f-4db6-ce73-00caca9e20a7"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lọc mẫu class 1\nX_c1 = X_test[y_test == 1]\nshap_c1 = shap_values[y_test == 1]\nshap.summary_plot(shap_c1, X_c1, max_display=X_test.shape[1])","metadata":{"id":"gErBM3_ONGyl","outputId":"f5dd4675-d3c9-4a9e-d857-b3cdb8055498"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Lọc mẫu có ground truth = 1\nmask_class1 = (y_test == 1)\nX_class1 = X_test[mask_class1]\nshap_class1 = shap_values[mask_class1]  # giữ đúng thứ tự dòng\n\n# 2. Đưa vào DataFrame để dễ thao tác\nshap_df = pd.DataFrame(shap_class1, columns=X_test.columns)\n\n# 3. Tính phần trăm số mẫu có SHAP > 0 cho mỗi feature\npct_positive_shap = (shap_df > 0).sum(axis=0) / shap_df.shape[0] * 100\n\n# 4. Đưa vào DataFrame kết quả\nresult_df = pd.DataFrame({\n    \"feature\": pct_positive_shap.index,\n    \"pct_SHAP_positive_in_class1\": pct_positive_shap.values\n}).sort_values(by=\"pct_SHAP_positive_in_class1\", ascending=False)\n\nresult_df","metadata":{"id":"RhVez3NBOSV1","outputId":"322f5478-4ca8-4086-dbde-36e320757012"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lọc mẫu class 0\nX_c0 = X_test[y_test == 0]\nshap_c0 = shap_values[y_test == 0]\nshap.summary_plot(shap_c0, X_c0, max_display=X_test.shape[1])","metadata":{"id":"G8MdX9uBNTQ0","outputId":"5cc3c858-a244-4c2c-8201-414179b1bc03"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Lọc các mẫu có label thực tế là 0\nmask_class0 = (y_test == 0)\nX_class0 = X_test[mask_class0]\nshap_class0 = shap_values[mask_class0]\n\n# 2. Đưa vào DataFrame\nshap_df = pd.DataFrame(shap_class0, columns=X_test.columns)\n\n# 3. Tính % số mẫu có SHAP > 0 trong class 0 (tức là feature đang đẩy nhầm về class 1)\npct_shap_positive = (shap_df > 0).sum(axis=0) / shap_df.shape[0] * 100\n\n# 4. Đưa vào bảng kết quả\nresult_df_2 = pd.DataFrame({\n    \"feature\": pct_shap_positive.index,\n    \"pct_SHAP_positive_in_class0\": pct_shap_positive.values\n}).sort_values(by=\"pct_SHAP_positive_in_class0\", ascending=False)\n\nresult_df_2","metadata":{"id":"2Rwai8rSPU5k","outputId":"1a304e4b-6e8f-44d7-daaf-e2971ce31fbd"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_value_report(df_10)","metadata":{"id":"xjvgZhJQywkj","outputId":"430c9646-aace-4c12-d1e0-d9da0a9e1aa6"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"cGslB48PofFN"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"_________","metadata":{"id":"u6oPOOArk9yl"}},{"cell_type":"code","source":"from catboost import Pool, CatBoostClassifier\n\ndef objective_CatBoost_Under(trial):\n    # Định nghĩa các tham số của CatBoost\n    params = {\n        \"iterations\": trial.suggest_int(\"iterations\", 100, 1000),\n        \"depth\": trial.suggest_int(\"depth\", 4, 10),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3),\n        \"l2_leaf_reg\": trial.suggest_float(\"l2_leaf_reg\", 1.0, 10.0),\n        \"border_count\": trial.suggest_int(\"border_count\", 32, 255),\n        # \"subsample\": trial.suggest_float(\"subsample\", 0.6, 1.0),\n        # \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.5, 1.0),\n        \"eval_metric\": \"F1\",\n        \"loss_function\": \"Logloss\",\n        \"random_seed\": 42,\n        \"verbose\": 0,\n        \"task_type\": \"GPU\"\n    }\n\n    # Tạo Pool cho CatBoost\n    train_pool = Pool(data=X_under, label=y_under, cat_features=categorical_features)\n    test_pool = Pool(data=X_test, label=y_test, cat_features=categorical_features)\n\n    # Khởi tạo và huấn luyện mô hình\n    model = CatBoostClassifier(**params)\n    model.fit(train_pool, eval_set=test_pool, use_best_model=True)\n\n    # Dự đoán và tính F1 score\n    y_pred = model.predict(X_test)\n    f1 = f1_score(y_test, y_pred)\n\n    return f1\n","metadata":{"id":"aowWIT4WxB-G"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_cb_under = optuna.create_study(direction='maximize')\nstudy_cb_under.optimize(objective_CatBoost_Under, n_trials=10)","metadata":{"id":"gT6_kJDZcYFL","outputId":"2c2d6496-ad71-465d-c4de-696afa4291b8"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best F1-score:\", study_cb_under.best_value)\nprint(\"Best parameters:\")\nfor k, v in study_cb_under.best_params.items():\n    print(f\"  {k}: {v}\")","metadata":{"id":"P2eaZFjacbQd","outputId":"55381450-0779-4731-e283-b2226537abb6"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_pool = Pool(data=X_under, label=y_under, cat_features=categorical_features)\n\nbest_params_cb_under = study_cb_under.best_params\nbest_model_cb_under = CatBoostClassifier(**best_params_cb_under)\nbest_model_cb_under.fit(train_pool)","metadata":{"id":"EgcVE7e9qYJ8","outputId":"4cd448d4-82a8-4215-f377-c98f636fc186"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Best model","metadata":{"id":"uBINtcwfZTzQ"}},{"cell_type":"code","source":"train_pool = Pool(data=X_under, label=y_under, cat_features=categorical_features)\n\nbest_model_cb_under = CatBoostClassifier(\n    iterations=601,\n    depth=9,\n    learning_rate=0.2048148456885514,\n    l2_leaf_reg=8.266595779431615,\n    border_count=240,\n    verbose=100 # Add verbose=0 to silence training output\n)\n\nbest_model_cb_under.fit(train_pool)","metadata":{"id":"z_yn4fk8ZWSl","outputId":"03c26d9f-b06c-4d1f-ce27-25bc35ff36a1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"explainer = shap.TreeExplainer(best_model_cb_under)\nshap_values = explainer.shap_values(X_test)","metadata":{"id":"bci0yj1ebCYE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"shap.summary_plot(shap_values, X_test, max_display=X_test.shape[1])","metadata":{"id":"xuvar9sibGTd"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# FINAL","metadata":{"id":"GgPSweiS1KWq"}},{"cell_type":"code","source":"X_train = df_not_9.drop(columns=['label'])\ny_train = df_not_9['label']\nX_test = df_9.drop(columns=['label'])\ny_test = df_9['label']","metadata":{"id":"q2sh89dQb88w"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_train = X_train[['day_range', 'SUPPLIER_CD', 'CUST_CD', 'VSD day', 'BRAND_CD', 'CLASSIFY_CD',\n#                   'DELI_DIV', 'PURCHASE AMOUNT', 'PRODUCT_CD', 'INNER_CD', 'SUPPLIER INV AMOUNT']]\n# X_test = X_test[['day_range', 'SUPPLIER_CD', 'CUST_CD', 'VSD day', 'BRAND_CD', 'CLASSIFY_CD',\n#                   'DELI_DIV', 'PURCHASE AMOUNT', 'PRODUCT_CD', 'INNER_CD', 'SUPPLIER INV AMOUNT']]","metadata":{"id":"MSABWEbb1Ums"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# categorical_features = ['SUPPLIER_CD', 'CUST_CD', 'BRAND_CD', 'CLASSIFY_CD', 'DELI_DIV', 'PRODUCT_CD', 'INNER_CD']","metadata":{"id":"gTPpN6bc3wxG"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = ['SUBSIDIARY_CD', 'CLASSIFY_CD', 'CUST_CD', 'BRAND_CD', 'INNER_CD', 'SUPPLIER_CD',\n                        'Sales order line number', 'Stock class', 'OTHER AREA SHIP DIV', 'PRODUCT_CD',\n                        'SPECIAL DIV', 'LOGICAL PLANT', 'DIRECT SHIP FLG',\n                        'DELI_DIV', 'Ship Mode', 'SUPPLIER_DIV', 'Is_Weekend', 'PACKING RANK']\n\nX_train['SUPPLIER_DIV'] = X_train['SUPPLIER_DIV'].astype(str)\nX_test['SUPPLIER_DIV'] = X_test['SUPPLIER_DIV'].astype(str)","metadata":{"id":"3-KtwLzl-R1-"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Trước khi undersample và oversample: {Counter(y_train)}\")\n\n# Calculate target counts based on desired ratios (majority:minority)\noriginal_minority_count = Counter(y_train)[1]\ndesired_majority_under = 17 * original_minority_count\ndesired_minority_under = original_minority_count\n\nunder_sampler = RandomUnderSampler(sampling_strategy={0: desired_majority_under, 1: desired_minority_under}, random_state=42)\nX_under, y_under = under_sampler.fit_resample(X_train, y_train)\nprint(f\"Sau khi under-sample (30:1 majority:minority): {Counter(y_under)}\")","metadata":{"id":"AE_3Rc6S1wHf","outputId":"59f481ea-80a2-43ed-e6c7-f797a6a86610"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostClassifier, Pool\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nimport optuna\nimport numpy as np\n\ndef objective_CatBoost_Under(trial):\n\n    X_tr, X_dev, y_tr, y_dev = train_test_split(\n        X_under, y_under,\n        test_size=0.20,          # bạn đổi tuỳ ý\n        stratify=y_under,\n        random_state=trial.number   # khác nhau mỗi trial\n    )\n\n    train_pool = Pool(X_tr,  y_tr, cat_features=categorical_features)\n    dev_pool   = Pool(X_dev, y_dev, cat_features=categorical_features)\n\n    # Định nghĩa các tham số của CatBoost\n    params = {\n        \"iterations\": trial.suggest_int(\"iterations\", 100, 1000),\n        \"depth\": trial.suggest_int(\"depth\", 4, 10),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3),\n        \"l2_leaf_reg\": trial.suggest_float(\"l2_leaf_reg\", 1.0, 10.0),\n        \"border_count\": trial.suggest_int(\"border_count\", 32, 255),\n        # \"subsample\": trial.suggest_float(\"subsample\", 0.6, 1.0),\n        # \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.5, 1.0),\n        \"eval_metric\": \"F1\",\n        \"loss_function\": \"Logloss\",\n        \"random_seed\": 42,\n        \"verbose\": 0,\n        \"task_type\": \"GPU\"\n    }\n\n\n    # Khởi tạo và huấn luyện mô hình\n    model = CatBoostClassifier(**params)\n    model.fit(train_pool, eval_set=dev_pool, early_stopping_rounds=80)\n\n    y_pred_dev = model.predict(X_dev)\n    f1_dev = f1_score(y_dev, y_pred_dev)\n\n    return f1_dev","metadata":{"id":"WKQ0HlGM2Nkm"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_cb_under = optuna.create_study(direction='maximize')\nstudy_cb_under.optimize(objective_CatBoost_Under, n_trials=5)","metadata":{"id":"5Hqoy-Sg3OC6","outputId":"c9ffe6b9-9393-45d7-994b-6377e88bdf77"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best F1-score:\", study_cb_under.best_value)\nprint(\"Best parameters:\")\nfor k, v in study_cb_under.best_params.items():\n    print(f\"  {k}: {v}\")","metadata":{"id":"duhcMaqY3WrO","outputId":"482d4371-3a97-4a2d-b15d-481b83d5fefc"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_pool = Pool(data=X_under, label=y_under, cat_features=categorical_features)\n\nbest_params_cb_under = study_cb_under.best_params\nbest_model_cb_under = CatBoostClassifier(**best_params_cb_under, task_type= \"GPU\", verbose=0)\nbest_model_cb_under.fit(train_pool)","metadata":{"id":"_bhAzl5y8yFt","outputId":"884b0f46-8293-40c6-cb01-5d1666cb8a4b"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = best_model_cb_under.predict(X_test)\nf1 = f1_score(y_test, y_pred)\nf1","metadata":{"id":"NmNOITfP8odo","outputId":"8d7c2a7e-67a2-42a8-b464-baa75a43b543"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"mBswrsRRFUtV"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_final = df_full.drop(columns=['label'])\ny_final = df_full['label']\nX_test_final = df_10","metadata":{"id":"ZSQrhmwx32ZY"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_final = X_final[['day_range', 'SUPPLIER_CD', 'CUST_CD', 'VSD day', 'BRAND_CD', 'CLASSIFY_CD',\n                  'DELI_DIV', 'PURCHASE AMOUNT', 'PRODUCT_CD', 'INNER_CD', 'SUPPLIER INV AMOUNT']]\nX_test_final = X_test_final[['day_range', 'SUPPLIER_CD', 'CUST_CD', 'VSD day', 'BRAND_CD', 'CLASSIFY_CD',\n                              'DELI_DIV', 'PURCHASE AMOUNT', 'PRODUCT_CD', 'INNER_CD', 'SUPPLIER INV AMOUNT']]","metadata":{"id":"3rae6ncz7KWr","outputId":"4ab14676-8073-48c5-d0f1-7403ec6c9729"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"gGBEzVsd7s8a"},"outputs":[],"execution_count":null}]}