{"metadata":{"kernelspec":{"display_name":"my_env","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.16"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":103436,"databundleVersionId":12476472,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"ce06556b","cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom catboost import CatBoostClassifier\nfrom sklearn.model_selection import StratifiedKFold\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{},"outputs":[],"execution_count":null},{"id":"bcaac18c","cell_type":"code","source":"data_4_6_delay = pd.read_csv(\"delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv\")\ndata_4_6_not_delay = pd.read_csv(\"not_delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv\")\ndata_7_9_delay = pd.read_csv(\"delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv\")\ndata_7_9_not_delay = pd.read_csv(\"not_delay_7_9_CONDITION_PRODUCT_SUPPLIER (1).csv\")\n","metadata":{},"outputs":[],"execution_count":null},{"id":"982aba3a","cell_type":"code","source":"data_4_6 = pd.concat([data_4_6_delay, data_4_6_not_delay], axis = 0, ignore_index= True)\ndata_7_9 = pd.concat([data_7_9_delay, data_7_9_not_delay], axis = 0, ignore_index= True)\nprint(data_4_6.info())\nprint(data_7_9.info())","metadata":{},"outputs":[],"execution_count":null},{"id":"8b62f760","cell_type":"code","source":"common_columns = data_4_6.columns.intersection(data_7_9.columns)\n\ncommon_columns","metadata":{},"outputs":[],"execution_count":null},{"id":"a876786f","cell_type":"code","source":"data_4_6 = data_4_6[common_columns]\ndata_7_9 = data_7_9[common_columns]","metadata":{},"outputs":[],"execution_count":null},{"id":"91b2ba2a","cell_type":"code","source":"data = pd.concat([data_4_6, data_7_9], axis = 0, ignore_index= True)","metadata":{},"outputs":[],"execution_count":null},{"id":"5300d92e","cell_type":"code","source":"data.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"3ebbdf3b","cell_type":"code","source":"data.describe()","metadata":{},"outputs":[],"execution_count":null},{"id":"7f83a5d1","cell_type":"markdown","source":"# Missing Value","metadata":{}},{"id":"6c1490c2","cell_type":"code","source":"for col in data.columns:\n    if data[col].isna().sum() > 0:\n        print(col, data[col].isna().sum())\n        data[col] = data[col].fillna('Unknow')","metadata":{},"outputs":[],"execution_count":null},{"id":"582be3c6","cell_type":"code","source":"data['Ship Mode'].value_counts()","metadata":{},"outputs":[],"execution_count":null},{"id":"bcb78a18","cell_type":"code","source":"data['SUPPLIER_DIV'].value_counts()","metadata":{},"outputs":[],"execution_count":null},{"id":"1283f157","cell_type":"code","source":"for col in data.columns:\n    print(col)\n    print(data[col].unique())","metadata":{},"outputs":[],"execution_count":null},{"id":"28611f36","cell_type":"markdown","source":"- Time Serie : Order Date, VSD, SO_TIME -> Count encode\n- Code / No : Classfy_cd, Cust_cd, Brand_cd, inner_cd, Supplier_cd, product_cd, SHIP DECISION NO, global no -> count encode\n- Numerical : Sales order line number, Consider Count holiday, So QTY, Allocation QTY, Supplier INV Amount, Purchase amount \n              PACK QTY, WEIGHT PER PIECE\n              SO_DAY_OF_MONTH\n              SO_DAY_OF_WEEK \n              \n\n- Categorical : PRODUCT ATTRIBUTION, SPECIAL DIV, LOGICAL PLANT -> Encode one-hot \n                DIRECT SHIP FLG -> one hot \n                DELI_DIV - tìm cách xử lý \n                Ship Mode -> label theo bảng alphabet\n                SUPPLIER_DIV (Phân loại nhà cung cấp) - là số nhưng được đánh theo thứ tự \n                SPECIAL_DIV\n\n\n\ncount encode : Order date\n\n","metadata":{}},{"id":"d73f39c3","cell_type":"markdown","source":"# Function ","metadata":{}},{"id":"1c6a6bd9","cell_type":"code","source":"\ndef histplot_func(data, col, bins):\n    sns.histplot(data = data, x = col, kde = True, bins = bins)\n    plt.show()\ndef count_encoding(data, col):\n    count_encoding = data[col].value_counts().to_dict()\n    data[col] = data[col].map(count_encoding)\n    #histplot_func(data, col, 100)\n\n# Tính Information Value\ndef calculate_woe_iv(df, feature, target):\n    # Nhóm dữ liệu theo bin, đếm Good và Bad\n    grouped = df.groupby(feature)[target].agg(['count', 'sum'])\n    grouped['Good'] = grouped['count'] - grouped['sum']  # Số mẫu Good\n    grouped['Bad'] = grouped['sum']  # Số mẫu Bad\n    \n    # Tính tỷ lệ\n    total_good = grouped['Good'].sum()\n    total_bad = grouped['Bad'].sum()\n    grouped['Pct_Good'] = grouped['Good'] / total_good\n    grouped['Pct_Bad'] = grouped['Bad'] / total_bad\n    \n    # Tính WoE\n    grouped['WoE'] = np.log(grouped['Pct_Good'] / grouped['Pct_Bad'].replace(0, 1e-6))\n    \n    # Tính IV\n    grouped['IV'] = (grouped['Pct_Good'] - grouped['Pct_Bad']) * grouped['WoE']\n    iv_total = grouped['IV'].sum()\n    \n    return grouped, iv_total\n\n\n# Hàm tính Null Importances\ndef get_null_importances(X, y, cat_features,  features, n_splits=5, n_shuffles=10):\n    # Khởi tạo K-Fold\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    # Ma trận lưu importance thực tế\n    real_importances = np.zeros(len(features))\n    \n    # Danh sách lưu null importances cho mỗi lần xáo trộn\n    null_importances = np.zeros((len(features), n_shuffles * n_splits))\n    \n    # Huấn luyện trên 5 folds\n    for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):\n        X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n        X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n        \n        # Khởi tạo CatBoostClassifier\n        model = CatBoostClassifier(\n            iterations=100,\n            scale_pos_weight=40,  # Xử lý mất cân bằng 1:40\n            random_seed=42,\n            verbose=0\n        )\n        \n        # Huấn luyện trên dữ liệu thực\n        model.fit(X_train, y_train, eval_set=(X_val, y_val), cat_features=cat_features)\n        real_importances += model.get_feature_importance() / n_splits\n        \n        # Tạo null importances\n        for shuffle_iter in range(n_shuffles):\n            X_shuffled = X_train.copy()\n            for col in features:\n                # Xáo trộn từng cột\n                X_shuffled[col] = np.random.permutation(X_shuffled[col].values)\n            \n            # Huấn luyện trên dữ liệu xáo trộn\n            model.fit(X_shuffled, y_train, eval_set=(X_val, y_val))\n            null_importances[:, fold * n_shuffles + shuffle_iter] = model.get_feature_importance()\n    \n    return real_importances, null_importances\n\n\n# Hàm chọn thuộc tính dựa trên Null Importances\ndef select_important_features(real_importances, null_importances, features, threshold_factor=2):\n    null_mean = null_importances.mean(axis=1)\n    null_std = null_importances.std(axis=1)\n    threshold = null_mean + threshold_factor * null_std\n    \n    selected_features = [\n        (features[i], real_importances[i]) \n        for i in range(len(features)) \n        if real_importances[i] > threshold[i]\n    ]\n    return selected_features\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"61484a8e","cell_type":"code","source":"#data.info()\n\n# Những thuộc tính categorical \n# VSD, Order Date, SO_TIME -> Count_encode \ndata = data.drop(columns= 'SUBSIDIARY_CD')\ncount_encoding_col = ['Order date', 'GLOBAL_NO', 'CLASSIFY_CD', 'CUST_CD', 'BRAND_CD',\n                      'INNER_CD', 'SUPPLIER_CD', 'PRODUCT_CD', 'SHIP DECISION NO', 'SOUF_RCV_NO', 'QTUF_RCV_NO',\n                      'VSD', 'REASON_CD', 'SO_TIME']\n\ncategorical_col = ['OTHER AREA SHIP DIV', 'DELI_DIV', 'Ship Mode', 'Stock class', 'Consider count hodiday Saturday',\n                   'PRODUCT ATTRIBUTION', 'SPECIAL DIV', 'LOGICAL PLANT', 'DIRECT SHIP FLG', 'SPECIAL_DIV', \n                   'PACKING RANK', 'SUPPLIER_DIV']\n\nnumerical_col = ['Sales order line number', 'SO QTY', 'ALLOCATION QTY', 'SUPPLIER INV AMOUNT', 'PURCHASE AMOUNT',\n                 'PACK QTY', 'WEIGHT PER PIECE', 'SO_DAY_OF_MONTH', 'SO_DAY_OF_WEEK']\n\n\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"a85954f9","cell_type":"code","source":"for col in numerical_col:\n    print(col)\n    print(data[col].unique())\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"23750f21","cell_type":"code","source":"data_iv = data.copy()\nfor col in categorical_col:\n    count_encoding(data_iv, col)\nfor col in count_encoding_col:\n    count_encoding(data_iv, col)","metadata":{},"outputs":[],"execution_count":null},{"id":"ab2ac8a2","cell_type":"code","source":"data_temp = pd.DataFrame(columns= ['Col_Name', 'Value'])\nfor col in data.columns:\n    result, iv = calculate_woe_iv(data_iv, col, 'label')\n    data_temp.loc[len(data_temp)] = [col, round(iv, 2)]\n\ndata_temp = data_temp.sort_values(by='Value', ascending=False)\n\n# Hiển thị bảng kết quả\nprint(\"\\nBảng IV sắp xếp từ lớn đến nhỏ:\")\ndata_temp\n","metadata":{},"outputs":[],"execution_count":null},{"id":"2784cf99","cell_type":"markdown","source":"# Correlation Matrix","metadata":{}},{"id":"167dd970","cell_type":"code","source":"corr_matrix = data_iv[categorical_col].corr()\nplt.figure(figsize=(10, 8))  # Kích thước biểu đồ\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', linewidths=0.5)\nplt.title('Correlation Matrix Heatmap Categorical')\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"317e42ec","cell_type":"code","source":"corr_matrix = data_iv[count_encoding_col].corr()\nplt.figure(figsize=(10, 8))  # Kích thước biểu đồ\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', linewidths=0.5)\nplt.title('Correlation Matrix Heatmap Categorical')\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"f8bea3ba","cell_type":"code","source":"corr_matrix = data_iv[numerical_col].corr()\nplt.figure(figsize=(10, 8))  # Kích thước biểu đồ\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', linewidths=0.5)\nplt.title('Correlation Matrix Heatmap Numerical')\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"95158428","cell_type":"markdown","source":"# Feature Selection ","metadata":{}},{"id":"4c306cb8","cell_type":"code","source":"\nX = data_iv.drop('label', axis=1)\ny = data_iv['label']\nfeatures = X.columns.tolist()\n\ncat_features = categorical_col + count_encoding_col\n\n# Tính Null Importances\nreal_importances, null_importances = get_null_importances(X, y, cat_features, features, n_splits=5, n_shuffles=10)\n\n# Tạo bảng kết quả\nimportance_df = pd.DataFrame({\n    'Feature': features,\n    'Real_Importance': real_importances,\n})\nimportance_df = importance_df.sort_values(by='Real_Importance', ascending=False)\n\n# Hiển thị kết quả\nprint(\"\\nBảng Feature Importance (sắp xếp từ lớn đến nhỏ):\")\nprint(importance_df.round(4).to_string(index=False))","metadata":{},"outputs":[],"execution_count":null},{"id":"fd9318a7","cell_type":"code","source":"importance_df","metadata":{},"outputs":[],"execution_count":null},{"id":"af292e14","cell_type":"markdown","source":"- Global no, stock class, so  day of week , so day of month, QTUF_RCV_NO, SOUF_RCV_NO, \n- purchase amount, ALLOCATION QTY bị tùng lặp\n- DIRECT SHIP FLG - đa cộng tuyến\n- REASON CD - lý do về mặt logic \n\n","metadata":{}},{"id":"73c32a57","cell_type":"code","source":"drop_col = [ 'BRAND_CD', 'ALLOCATION QTY', 'SPECIAL DIV', , 'Sales order line number','LOGICAL PLANT', \n            'PURCHASE AMOUNT', 'DIRECT SHIP FLG', 'SHIP DECISION NO', 'PACK QTY','GLOBAL_NO', \n            'SUPPLIER_DIV', 'SPECIAL_DIV', 'SO_DAY_OF_MONTH', 'SO_DAY_OF_WEEK', 'SO_TIME', 'QTUF_RCV_NO', \n            'SOUF_RCV_NO',, 'Stock class', 'Order date', 'VSD', 'SO_TIME', 'REASON_CD']\ndata_t = data.copy()\n\ndata_t = data_t.drop(columns= drop_col)","metadata":{},"outputs":[],"execution_count":null},{"id":"7739131f","cell_type":"code","source":"cat_features","metadata":{},"outputs":[],"execution_count":null},{"id":"e9813b46","cell_type":"code","source":"# Thay thế dấu cách và xử lý NaN\ndata_t = data_t.replace(' ', 'Unknow')\n\n# Chuyển các cột phân loại thành chuỗi\nfor col in cat_features:\n    if col not in drop_col:\n        data_t[col] = data_t[col].astype(str)","metadata":{},"outputs":[],"execution_count":null},{"id":"4784c715","cell_type":"markdown","source":"# Undersampling","metadata":{}},{"id":"6a75c9d3","cell_type":"code","source":"from sklearn.utils import resample\n\n# Hàm thực hiện undersampling\ndef undersample_data(X, y, majority_class_size, random_state=42):\n    X_majority = X[y == 0]\n    y_majority = y[y == 0]\n    X_minority = X[y == 1]\n    y_minority = y[y == 1]\n    \n    X_majority_downsampled, y_majority_downsampled = resample(\n        X_majority, y_majority,\n        n_samples=majority_class_size,\n        random_state=random_state,\n        replace=False\n    )\n    \n    X_balanced = pd.concat([X_majority_downsampled, X_minority], axis=0)\n    y_balanced = pd.concat([y_majority_downsampled, y_minority], axis=0)\n    \n    return X_balanced, y_balanced\n\n# Chuẩn bị dữ liệu\nX = data_t.drop('label', axis=1)\ny = data_t['label']\n\nn_minority = sum(y == 1) \nn_majority = n_minority * 20 \nX_balanced, y_balanced = undersample_data(X, y, n_majority, random_state=42)\n\n# Kiểm tra kết quả\nprint(f\"Số mẫu sau undersampling: {len(y_balanced)}\")\nprint(f\"Tỷ lệ lớp (not_delay:delay): {sum(y_balanced == 0)}:{sum(y_balanced == 1)}\")\n","metadata":{},"outputs":[],"execution_count":null},{"id":"0640011b","cell_type":"code","source":"for col in categorical_col:\n    if col not in drop_col:\n        data_t[col] = data_t[col].astype('category')\nfor col in count_encoding_col:\n    if col not in drop_col:\n        data_t[col] = data_t[col].astype('category')\n    ","metadata":{},"outputs":[],"execution_count":null},{"id":"d4a0b2e1","cell_type":"code","source":"data_t.info()","metadata":{},"outputs":[],"execution_count":null},{"id":"de7b3780","cell_type":"markdown","source":"# Model","metadata":{}},{"id":"06d04378","cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom catboost import CatBoostClassifier, Pool\n\nX = X_balanced\ny = y_balanced\n\n# 1. Chia tập train (60%) và tạm (40%)\nX_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.4, random_state=42, stratify=y)\n\n# 2. Chia tập tạm thành dev (20%) và test (20%)\nX_dev, X_test, y_dev, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp)\n\n# Xác định các cột phân loại\ncat_features = X.select_dtypes(include=['category', 'object']).columns.tolist()\n\ncat_features\n","metadata":{},"outputs":[],"execution_count":null},{"id":"c7573839","cell_type":"code","source":"train_pool = Pool(data=X_train, label=y_train, cat_features=cat_features)\ndev_pool = Pool(data=X_dev, label=y_dev, cat_features=cat_features)\ntest_pool = Pool(data=X_test, label=y_test, cat_features=cat_features)\n\n# Huấn luyện mô hình\nmodel = CatBoostClassifier(verbose=100, learning_rate= 0.1, n_estimators= 1000, random_seed= 42, early_stopping_rounds= 100)\nmodel.fit(train_pool, eval_set=dev_pool)","metadata":{},"outputs":[],"execution_count":null},{"id":"2bbe7587","cell_type":"code","source":"'''\nfrom sklearn.model_selection import RandomizedSearchCV\n\nmodel = CatBoostClassifier(\n    verbose=100,\n    iterations=1000,\n    learning_rate=0.1,\n    random_seed=42,\n    cat_features=cat_features\n    early_stopping_rounds=100\n)\n\nparam_dist = {\n    'depth': [4, 6, 8, 10],\n    'l2_leaf_reg': [1, 3, 5, 7, 9],\n    'bagging_temperature': [0, 0.3, 0.6, 1],\n}\n\nrandom_search = RandomizedSearchCV(\n    estimator=model,\n    param_distributions=param_dist,\n    n_iter=20,  # Số mẫu thử\n    scoring='f1_macro',  \n    cv=5,\n    random_state=42,\n    n_jobs=-1\n)\n\nrandom_search.fit(X_train, y_train)\nprint(\"Best params:\", random_search.best_params_)\n'''","metadata":{},"outputs":[],"execution_count":null},{"id":"5ebb26f6","cell_type":"code","source":"from sklearn.metrics import classification_report\n\ny_pred = model.predict(X_test)\nprint(classification_report(y_test, y_pred))","metadata":{},"outputs":[],"execution_count":null},{"id":"55f5017a","cell_type":"code","source":"X_balanced","metadata":{},"outputs":[],"execution_count":null},{"id":"c63e0f80","cell_type":"code","source":"data_pilot = pd.read_csv('test.csv').drop(columns = 'ID')\ndata_pilot","metadata":{},"outputs":[],"execution_count":null},{"id":"76c1a373","cell_type":"code","source":"print(data_pilot.columns[13])","metadata":{},"outputs":[],"execution_count":null},{"id":"7ac26b7f","cell_type":"code","source":"X_balanced['WEIGHT PER PIECE'].dtype","metadata":{},"outputs":[],"execution_count":null},{"id":"71f8f182","cell_type":"code","source":"cat_features","metadata":{},"outputs":[],"execution_count":null},{"id":"030542a3","cell_type":"code","source":"for col in data_pilot.columns:\n    if col not in cat_features:\n        print(col, data_pilot[col].unique())","metadata":{},"outputs":[],"execution_count":null},{"id":"0c36b1a1","cell_type":"code","source":"data_pilot['WEIGHT PER PIECE'] = data_pilot['WEIGHT PER PIECE'].replace('Unknow', 0)\ndata_pilot['WEIGHT PER PIECE'] = data_pilot['WEIGHT PER PIECE'].astype(float)","metadata":{},"outputs":[],"execution_count":null},{"id":"38779586","cell_type":"code","source":"y_pred = model.predict(data_pilot)","metadata":{},"outputs":[],"execution_count":null},{"id":"02039f5b","cell_type":"code","source":"submit = pd.DataFrame(columns = ['ID', 'label'])\nsubmit['label'] = y_pred\nsubmit['ID'] = range(1, len(y_pred) + 1)","metadata":{},"outputs":[],"execution_count":null},{"id":"5f559b9a","cell_type":"code","source":"submit.to_csv('submission.csv', index=False)","metadata":{},"outputs":[],"execution_count":null},{"id":"d5ae273c","cell_type":"code","source":"submit","metadata":{},"outputs":[],"execution_count":null}]}