{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":103436,"databundleVersionId":12476472,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndata_delay_4_6 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv')\ndata_delay_7_9 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv')\n\ndata_not_delay_4_6 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/not_delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv')\ndata_not_delay_7_9 = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/not_delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv')\n\npilot = pd.read_csv('/kaggle/input/ds-108-p-21-assigment-06/PILOT_10.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(data_delay_4_6.columns), len(data_delay_7_9.columns), len(data_not_delay_4_6.columns), len(data_not_delay_7_9.columns), len(pilot.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_delay_4_6.columns.difference(data_delay_7_9.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_delay_7_9.columns.difference(pilot.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_delay_4_6.columns.difference(pilot.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_not_delay_4_6.columns.difference(data_delay_4_6.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"diff_features = ['ACTUAL_SHIP_DAYS', 'EXPENSIVE_FLG', 'HAZARD_FLG', 'HEAVY_FLG',\n       'IO_UNFIT_FLG', 'PRODUCT_ASSORT', 'SPECIFY_PRODUCTION_DAYS',\n       'SPECIFY_SHIP_DAYS', 'SUPPLIER_CATEGORY_CD', 'WEIGHT_UNIT']\n\ndata_delay_4_6 = data_delay_4_6.drop(columns=diff_features)\ndata_not_delay_4_6 = data_not_delay_4_6.drop(columns=diff_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(data_delay_4_6.columns), len(data_delay_7_9.columns), len(data_not_delay_4_6.columns), len(data_not_delay_7_9.columns), len(pilot.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Concatenation","metadata":{}},{"cell_type":"code","source":"data_4_6 = pd.concat([data_delay_4_6, data_not_delay_4_6], axis=0)\ndata_4_6 = data_4_6.sample(frac=1, random_state=42)\ndata_4_6.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_7_9 = pd.concat([data_delay_7_9, data_not_delay_7_9], axis=0)\ndata_7_9 = data_7_9.sample(frac=1, random_state=42)\ndata_7_9.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data = pd.concat([data_4_6, data_7_9], axis=0)\nfull_data = full_data.sample(frac=1, random_state=42)\nfull_data","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing Values","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nmissing_count = full_data.isnull().sum()\nmissing_count = missing_count[missing_count > 0]\nmissing_info = pd.DataFrame({'missing': missing_count,'percentage': (missing_count / len(full_data)) * 100})\nmissing = missing_info.sort_values(by='missing', ascending=True)\n\nplt.figure(figsize=(10, 6))\nax = missing['missing'].plot(kind='barh', color='orange')\nplt.grid(axis='x')\nplt.xticks(rotation=0)\nplt.yticks(rotation=0)\nplt.title('Missing values in data')\nplt.xlabel('Number of missing values')\nplt.ylabel('Columns')\n\nfor i, (val, pct) in enumerate(zip(missing['missing'], missing_info['percentage'])):\n    ax.text(val + len(full_data)*0.002, i, f'{pct:.2f}%', va='center', fontsize=10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drop_cols = missing[missing['percentage'] > 50].index","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data = full_data.drop(columns=drop_cols)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in missing_info[missing_info['percentage'] < 50].index:\n    print(f\"Column {col} has {missing_info['missing'][col]} missing values, which is {missing_info['percentage'][col]:.2f}% of the total data\")\n    print(f\"Data type: {full_data[col].dtype}\")\n    print(f\"Unique values: {full_data[col].unique()}\\n\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data.dropna(subset=['SUPPLIER_DIV'], inplace=True)\nfull_data['Ship Mode'] = full_data['Ship Mode'].fillna(full_data['Ship Mode'].mode()[0])\nfull_data['SHIP DECISION NO'] = full_data['SHIP DECISION NO'].fillna(-1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_count = pilot.isnull().sum()\nmissing_count = missing_count[missing_count > 0]\nmissing_info = pd.DataFrame({'missing': missing_count,'percentage': (missing_count / len(pilot)) * 100})\nmissing = missing_info.sort_values(by='missing', ascending=True)\n\nplt.figure(figsize=(10, 6))\nax = missing['missing'].plot(kind='barh', color='orange')\nplt.grid(axis='x')\nplt.xticks(rotation=0)\nplt.yticks(rotation=0)\nplt.title('Missing values in data')\nplt.xlabel('Number of missing values')\nplt.ylabel('Columns')\n\nfor i, (val, pct) in enumerate(zip(missing['missing'], missing_info['percentage'])):\n    ax.text(val + len(pilot)*0.002, i, f'{pct:.2f}%', va='center', fontsize=10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drop_cols = missing[missing['percentage'] > 50].index","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pilot = pilot.drop(columns=drop_cols)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in missing_info[missing_info['percentage'] < 50].index:\n    print(f\"Column {col} has {missing_info['missing'][col]} missing values, which is {missing_info['percentage'][col]:.2f}% of the total data\")\n    print(f\"Data type: {pilot[col].dtype}\")\n    print(f\"Unique values: {pilot[col].unique()}\\n\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pilot['SUPPLIER_DIV'] = pilot['SUPPLIER_DIV'].fillna(-1)\npilot['Ship Mode'] = pilot['Ship Mode'].fillna(pilot['Ship Mode'].mode()[0])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Invalid values cheking","metadata":{}},{"cell_type":"code","source":"invalid_values = ['', ' ', None]\n\nfor col in full_data.columns:\n    if full_data[col].isin(invalid_values).any():\n        print(f\"Cột '{col}' chứa giá trị rỗng: {full_data[col][full_data[col].isin(invalid_values)].unique()}\")\nfull_data = full_data.replace(['', ' ', None], -1).fillna(-1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in pilot.columns:\n    if pilot[col].isin(invalid_values).any():\n        print(f\"Cột '{col}' chứa giá trị rỗng: {pilot[col][pilot[col].isin(invalid_values)].unique()}\")\npilot = pilot.replace(['', ' ', None], -1).fillna(-1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data.columns.difference(pilot.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data = full_data.drop(columns='SHIP DECISION NO')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Retype","metadata":{}},{"cell_type":"markdown","source":"## Date time ","metadata":{}},{"cell_type":"code","source":"full_data['VSD'] = pd.to_datetime(full_data['VSD'], format='mixed')\nfull_data['Order date'] = pd.to_datetime(full_data['Order date'], format='mixed')\n\nfull_data['VSD'] = full_data['VSD'].dt.date\nfull_data['Order date'] = full_data['Order date'].dt.date\n\nfull_data['VSD'] = pd.to_datetime(full_data['VSD'], errors='coerce', infer_datetime_format=True)\nfull_data['Order date'] = pd.to_datetime(full_data['Order date'], errors='coerce', infer_datetime_format=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pilot['VSD'] = pd.to_datetime(pilot['VSD'], format='mixed')\npilot['Order date'] = pd.to_datetime(pilot['Order date'], format='mixed')\n\npilot['VSD'] = pilot['VSD'].dt.date\npilot['Order date'] = pilot['Order date'].dt.date\n\npilot['VSD'] = pd.to_datetime(pilot['VSD'], errors='coerce', infer_datetime_format=True)\npilot['Order date'] = pd.to_datetime(pilot['Order date'], errors='coerce', infer_datetime_format=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"date_time_cols = ['VSD', 'Order date']\nfor col in date_time_cols:\n    full_data[f'day_of_week_{col}'] = full_data[col].dt.dayofweek\n    full_data[f'month_{col}'] = full_data[col].dt.month\n    full_data[f'week_{col}'] = full_data[col].dt.isocalendar().week\n\nfor col in date_time_cols:\n    pilot[f'day_of_week_{col}'] = pilot[col].dt.dayofweek\n    pilot[f'month_{col}'] = pilot[col].dt.month\n    pilot[f'week_{col}'] = pilot[col].dt.isocalendar().week","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data.columns.difference(pilot.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_type = {\n    'SUBSIDIARY_CD': 'str',\n    'GLOBAL_NO': 'str',\n    'CLASSIFY_CD': 'str',\n    'CUST_CD': 'str',\n    'BRAND_CD': 'str',\n    'INNER_CD': 'str',\n    'SUPPLIER_CD': 'str',\n    'Sales order line number': 'int64',\n    'Stock class': 'str',\n    'Consider count hodiday Saturday': 'int64',\n    'SO QTY': 'int64',\n    'ALLOCATION QTY': 'int64',\n    'SUPPLIER INV AMOUNT': 'float64',\n    'PACKING RANK': 'str',\n    'PRODUCT_CD': 'str',\n    'PRODUCT ATTRIBUTION': 'str',\n    'SPECIAL DIV': 'str',\n    'LOGICAL PLANT': 'str',\n    'PURCHASE AMOUNT': 'float64',\n    'DIRECT SHIP FLG': 'str',\n    'DELI_DIV': 'str',\n    'Ship Mode': 'str',\n    'PACK QTY': 'int64',\n    'WEIGHT PER PIECE': 'float64',\n    'SUPPLIER_DIV': 'str',\n    'SPECIAL_DIV': 'str',\n    'SO_DAY_OF_MONTH': 'int64', \n    'SO_DAY_OF_WEEK': 'int64', \n    'SO_TIME': 'int64'\n}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col, col_type in cols_type.items():\n    if col in full_data.columns:\n        full_data[col] = full_data[col].astype(col_type)\n\nfor col, col_type in cols_type.items():\n    if col in pilot.columns:\n        pilot[col] = pilot[col].astype(col_type)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"markdown","source":"## Corr matrix","metadata":{}},{"cell_type":"markdown","source":"### Numerical features","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n\nnumerical_features = [\n    'Sales order line number',\n    'Consider count hodiday Saturday',\n    'SO QTY',\n    'ALLOCATION QTY',\n    'SUPPLIER INV AMOUNT',\n    'PURCHASE AMOUNT',\n    'PACK QTY',\n    'WEIGHT PER PIECE',\n    'SO_DAY_OF_MONTH', \n    'SO_DAY_OF_WEEK', \n    'SO_TIME'\n]\n\nplt.figure(figsize=(10, 6))\nnumerical_data = full_data[numerical_features].corr()\nsns.heatmap(numerical_data, annot=True, fmt='.4f', cmap='Blues', mask=np.triu(numerical_data.corr()))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = full_data[numerical_features].corr().abs()\nupper_tri = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\nhigh_corr_pairs = [\n    (row, col, upper_tri.loc[row, col])\n    for row in upper_tri.index\n    for col in upper_tri.columns\n    if upper_tri.loc[row, col] > 0.9\n]\nlabel_corr = full_data[numerical_features + ['label']].corr()['label'].abs().drop('label')\nto_drop = set()\nfor row, col, corr_val in high_corr_pairs:\n    if label_corr[col] > label_corr[row]:\n        if row != \"SO QTY\":\n            to_drop.add(row)\n    else:\n        if col != \"SO QTY\":\n            to_drop.add(col)\n\nto_drop = list(to_drop)\nfull_data = full_data.drop(columns=to_drop)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nnumerical_data = pilot[numerical_features].corr()\nsns.heatmap(numerical_data, annot=True, fmt='.4f', cmap='Blues', mask=np.triu(numerical_data.corr()))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drop = ['ALLOCATION QTY', 'PURCHASE AMOUNT']\npilot = pilot.drop(columns=drop)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Categorical features","metadata":{}},{"cell_type":"code","source":"def frequency_encoding(df, cat_cols):\n    df = df.copy()\n    for col in cat_cols:\n        freq = df[col].value_counts() / len(df)\n        df[col] = df[col].map(freq)\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = [col for col, col_type in cols_type.items() if col_type == 'str']\nprint(categorical_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_data_cat_fq = frequency_encoding(full_data, categorical_features)\npilot_cat_fq = frequency_encoding(pilot, categorical_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_corr = full_data_cat_fq[categorical_features].corr()\nmask = np.triu(np.ones_like(cat_corr, dtype=bool))\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(cat_corr, mask=mask, annot=True, fmt=\".2f\", cmap=\"coolwarm\")\nplt.title(\"Correlation heatmap of categorical features\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = full_data_cat_fq[categorical_features].corr().abs()\nupper_tri = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n\nlabel_corr = full_data_cat_fq[categorical_features + ['label']].corr()['label'].abs().drop('label')\n\nto_drop = set()\ndropped_pairs = []\n\nfor col in upper_tri.columns:\n    for row in upper_tri.index:\n        if upper_tri.loc[row, col] > 0.9:\n            dropped_pairs.append((row, col, upper_tri.loc[row, col]))\n            if label_corr.get(col, 0) >= label_corr.get(row, 0):\n                to_drop.add(row)\n            else:\n                to_drop.add(col)\n\n# Drop \nfull_data_cat_fq = full_data_cat_fq.drop(columns=list(to_drop))\n\n# Update categorical_features\ncategorical_features = [col for col in categorical_features if col not in to_drop]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for row, col, corr_value in dropped_pairs:\n    if row in to_drop or col in to_drop:\n        print(f\"Dropped one of: ({row}, {col}) | corr = {corr_value:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_corr = pilot_cat_fq[categorical_features].corr()\nmask = np.triu(np.ones_like(cat_corr, dtype=bool))\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(cat_corr, mask=mask, annot=True, fmt=\".2f\", cmap=\"coolwarm\")\nplt.title(\"Correlation heatmap of categorical features\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pilot_cat_fq = pilot_cat_fq.drop(columns=['DELI_DIV', 'SPECIAL DIV', 'Stock class'])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Total Weight = 'WEIGHT PER PIECE' * 'SO QTY'\nfull_data_cat_fq['WEIGHT'] = full_data_cat_fq['WEIGHT PER PIECE'] * full_data_cat_fq['SO QTY']\npilot_cat_fq['WEIGHT'] = pilot_cat_fq['WEIGHT PER PIECE'] * pilot_cat_fq['SO QTY']\n\n# Day range = 'VSD' - 'Order date'\nfull_data_cat_fq['day_range'] = (pd.to_datetime(full_data_cat_fq['VSD']) - pd.to_datetime(full_data_cat_fq['Order date'])).dt.days\npilot_cat_fq['day_range'] = (pd.to_datetime(pilot_cat_fq['VSD']) - pd.to_datetime(pilot_cat_fq['Order date'])).dt.days\n\n# Drop 'WEIGHT PER PIECE', 'SO QTY', 'VSD', 'Order date'\ndrop_weight_features = ['WEIGHT PER PIECE', 'SO QTY', 'VSD', 'Order date']\nfull_data_cat_fq = full_data_cat_fq.drop(columns=drop_weight_features)\npilot_cat_fq = pilot_cat_fq.drop(columns=drop_weight_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pilot_cat_fq.shape, full_data_cat_fq.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Noise","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import IsolationForest\n\niso = IsolationForest(contamination=0.0001, random_state=42) \npreds = iso.fit_predict(full_data_cat_fq)\n\noutlier_idx = full_data_cat_fq[preds == -1].index\nprint(f\"Drop points: {len(outlier_idx)}\")\n\nfull_data_cat_fq = full_data_cat_fq.drop(index=outlier_idx)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"## Imblance","metadata":{}},{"cell_type":"code","source":"label = full_data_cat_fq['label']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\n\ncounts = label.value_counts()\npercentages = (counts / counts.sum() * 100).round(2)\ntotal = counts.sum()\n\nax = counts.plot(kind='bar', color=['r', 'y'])\n\nfor i, (count, pct) in enumerate(zip(counts, percentages)):\n    ax.text(i, count + total * 0.01, f'{pct}%', ha='center', fontsize=12)\n\nplt.xlabel('Label')\nplt.ylabel('Frequency')\nplt.xticks(rotation=0)\nplt.title('Label Distribution')\nplt.grid(axis='y', linestyle='--')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Undersamling","metadata":{}},{"cell_type":"code","source":"from sklearn.utils import resample\n\nlabel_counts = full_data_cat_fq['label'].value_counts()\nmajority_class = label_counts.idxmax()\nminority_class = label_counts.idxmin()\n\ndf_majority = full_data_cat_fq[full_data_cat_fq['label'] == majority_class]\ndf_minority = full_data_cat_fq[full_data_cat_fq['label'] == minority_class]\n\ndf_majority_downsampled = resample(\n    df_majority,\n    replace=False,\n    n_samples=len(df_minority),\n    random_state=42\n)\n\ndf_undersampled = pd.concat([df_majority_downsampled, df_minority])\ndf_undersampled = df_undersampled.sample(frac=1, random_state=42).reset_index(drop=True)\n\nprint(\"After undersampling:\")\nprint(df_undersampled['label'].value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Oversampling","metadata":{}},{"cell_type":"code","source":"from sklearn.utils import resample\n\nlabel_counts = full_data_cat_fq['label'].value_counts()\nmajority_class = label_counts.idxmax()\nminority_class = label_counts.idxmin()\n\ndf_majority = full_data_cat_fq[full_data_cat_fq['label'] == majority_class]\ndf_minority = full_data_cat_fq[full_data_cat_fq['label'] == minority_class]\n\ndf_minority_oversampled = resample(\n    df_minority,\n    replace=True,  \n    n_samples=len(df_majority),  \n    random_state=42\n)\n\ndf_oversampled = pd.concat([df_majority, df_minority_oversampled])\ndf_oversampled = df_oversampled.sample(frac=1, random_state=42).reset_index(drop=True)\n\nprint(\"After oversampling:\")\nprint(df_oversampled['label'].value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Voting","metadata":{}},{"cell_type":"code","source":"pip install lightgbm xgboost catboost optuna","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.svm import SVC\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Undersampling\ndf_train_un, df_temp = train_test_split(df_undersampled, test_size=0.2, random_state=42, stratify=df_undersampled['label'])\ndf_val_un, df_test_un = train_test_split(df_temp, test_size=0.5, random_state=42, stratify=df_temp['label'])\n\n# Oversampling\ndf_train_ov, df_temp = train_test_split(df_oversampled, test_size=0.2, random_state=42, stratify=df_oversampled['label'])\ndf_val_ov, df_test_ov = train_test_split(df_temp, test_size=0.5, random_state=42, stratify=df_temp['label'])\n\nprint(\"=====UNDERSAMPLING=====\")\nprint(\"Train size:\", df_train_un.shape)\nprint(\"Validation size:\", df_val_un.shape)\nprint(\"Test size:\", df_test_un.shape)\n\nprint(\"Label distribution in train:\")\nprint(df_train_un['label'].value_counts())\n\n\nprint(\"=====OVERSAMPLING=====\")\nprint(\"Train size:\", df_train_ov.shape)\nprint(\"Validation size:\", df_val_ov.shape)\nprint(\"Test size:\", df_test_ov.shape)\n\nprint(\"Label distribution in train:\")\nprint(df_train_ov['label'].value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Configs","metadata":{}},{"cell_type":"code","source":"def LightGBM(trial, X_train, y_train, X_val, y_val):\n    params = {\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 1000),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 200),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 12),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n        \"subsample\": trial.suggest_float(\"subsample\", 0.5, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 1.0),\n    }\n    model = lgb.LGBMClassifier(**params)\n    model.fit(X_train, y_train)\n    preds = model.predict(X_val)\n    return f1_score(y_val, preds, average='weighted'), params\n\n# XGBoost\ndef XGBoost(trial, X_train, y_train, X_val, y_val):\n    params = {\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 1000),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 12),\n        \"subsample\": trial.suggest_float(\"subsample\", 0.5, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 1.0),\n        \"gamma\": trial.suggest_float(\"gamma\", 0, 5),\n        \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0.01, 10.0),\n        \"enable_categorical\": True,\n        \"use_label_encoder\": False,\n        \"eval_metric\": 'logloss'\n    }\n    model = xgb.XGBClassifier(**params)\n    model.fit(X_train, y_train)\n    preds = model.predict(X_val)\n    return f1_score(y_val, preds, average='weighted'), params\n\n# CatBoost\ndef CatBoost(trial, X_train, y_train, X_val, y_val):\n    params = {\n        \"iterations\": trial.suggest_int(\"iterations\", 100, 1000),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3),\n        \"depth\": trial.suggest_int(\"depth\", 3, 10),\n        \"l2_leaf_reg\": trial.suggest_float(\"l2_leaf_reg\", 1e-3, 10.0),\n        \"random_strength\": trial.suggest_float(\"random_strength\", 1e-3, 10.0),\n        \"verbose\": 0,\n        \"random_seed\": 42\n    }\n    model = CatBoostClassifier(**params)\n    model.fit(X_train, y_train)\n    preds = model.predict(X_val)\n    return f1_score(y_val, preds, average='weighted'), params\n\n# SVM\ndef SVM(trial, X_train, y_train, X_val, y_val):\n    params = {\n        \"C\": trial.suggest_float(\"C\", 0.1, 10.0, log=True),\n        \"kernel\": trial.suggest_categorical(\"kernel\", [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]),\n        \"gamma\": trial.suggest_categorical(\"gamma\", [\"scale\", \"auto\"]),\n    }\n    model = SVC(**params)\n    model.fit(X_train, y_train)\n    preds = model.predict(X_val)\n    return f1_score(y_val, preds, average='weighted'), params","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Undersampling exp","metadata":{}},{"cell_type":"code","source":"# Objective wrapper\ndef objective(model_func, X_train, y_train, X_val, y_val):\n    def inner(trial):\n        val_f1, params = model_func(trial, X_train, y_train, X_val, y_val)\n        trial.set_user_attr(\"best_params\", params)\n        return val_f1\n    return inner\n\n# Cross validation\nX_train = df_train_un.drop(columns=['label'])\ny_train = df_train_un['label']\nX_val = df_val_un.drop(columns=['label'])\ny_val = df_val_un['label']\nX_test = df_test_un.drop(columns=['label'])\ny_test = df_test_un['label']\n\nmodels = {\n    \"LightGBM\": (LightGBM, lgb.LGBMClassifier),\n    # \"XGBoost\": (XGBoost, xgb.XGBClassifier),\n    # \"CatBoost\": (CatBoost, CatBoostClassifier),\n    # \"SVM\": (SVM, SVC)\n}\n\nresults = []\n\nfor name, (func, cls) in models.items():\n    print(f\"Tuning {name}...\")\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(objective(func, X_train, y_train, X_val, y_val), n_trials=5)\n    \n    best_params = study.best_trial.user_attrs['best_params']\n    print(f\"Best Params for {name}:\", best_params)\n\n    model = cls(**best_params)\n    model.fit(pd.concat([X_train, X_val]), pd.concat([y_train, y_val]))\n    preds = model.predict(X_test)\n\n    results.append({\n        'Model': name,\n        'Val F1': study.best_value,\n        'Test F1': f1_score(y_test, preds, average='macro'),\n        'Test Acc': accuracy_score(y_test, preds),\n        'Best Params': best_params\n    })\n\n    # Submission\n    pilot_preds = model.predict(pilot_cat_fq.drop(columns='ID'))\n    submission = pd.DataFrame({\n        'ID': pilot_cat_fq['ID'].values,\n        'label': pilot_preds\n    })\n    submission.to_csv(f\"submission.csv\", index=False)\n\n# Kết quả tổng hợp\nresults_df = pd.DataFrame(results)\nprint(results_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Oversampling exp","metadata":{}},{"cell_type":"code","source":"# # Cross validation\n# X_train_ov = df_train_ov.drop(columns=['label'])\n# y_train_ov = df_train_ov['label']\n# X_val_ov = df_val_ov.drop(columns=['label'])\n# y_val_ov = df_val_ov['label']\n# X_test_ov = df_test_ov.drop(columns=['label'])\n# y_test_ov = df_test_ov['label']\n# models = {\n#     \"LightGBM\": (LightGBM, lgb.LGBMClassifier),\n#     # \"XGBoost\": (XGBoost, xgb.XGBClassifier),\n#     # \"CatBoost\": (CatBoost, CatBoostClassifier),\n#     # \"SVM\": (SVM, SVC)\n# }\n# results_ov = []\n# for name, (func, cls) in models.items():\n#     print(f\"Tuning {name}...\")\n#     study = optuna.create_study(direction=\"maximize\")\n#     study.optimize(objective(func, X_train_ov, y_train_ov, X_val_ov, y_val_ov), n_trials=5)\n    \n#     best_params = study.best_trial.user_attrs['best_params']\n#     print(f\"Best Params for {name}:\", best_params)\n#     model = cls(**best_params)\n#     model.fit(pd.concat([X_train_ov, X_val_ov]), pd.concat([y_train_ov, y_val_ov]))\n#     preds = model.predict(X_test_ov)\n#     results_ov.append({\n#         'Model': name,\n#         'Val F1': study.best_value,\n#         'Test F1': f1_score(y_test_ov, preds, average='macro'),\n#         'Test Acc': accuracy_score(y_test_ov, preds),\n#         'Best Params': best_params\n#     })\n#     # Submission\n#     pilot_preds = model.predict(pilot_cat_fq.drop(columns='ID'))\n#     submission = pd.DataFrame({\n#         'ID': pilot_cat_fq['ID'].values,\n#         'label': pilot_preds\n#     })\n#     submission.to_csv(f\"submission_{name}_ov.csv\", index=False)\n# # Kết quả tổng hợp\n# results_ov_df = pd.DataFrame(results_ov)\n# print(results_ov_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# results_df = pd.DataFrame(results)\n# best_model_row = results_df.loc[results_df['Test F1'].idxmax()]\n# best_model_name = best_model_row['Model']\n# best_model_params = best_model_row['Best Params']\n\n# # Best model\n# if best_model_name == 'LightGBM':\n#     best_model = lgb.LGBMClassifier(**best_model_params)\n# elif best_model_name == 'XGBoost':\n#     best_model = xgb.XGBClassifier(**best_model_params)\n# elif best_model_name == 'CatBoost':\n#     best_model = CatBoostClassifier(**best_model_params, verbose=0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# best_model.fit(X_train, y_train)\n# pilot_preds = best_model.predict(df_pilot_pca.drop(columns='ID'))\n# submission = pd.DataFrame({\n#     'ID': df_pilot_pca['ID'].values,\n#     'label': pilot_preds\n# })\n# submission.to_csv(\"submission1.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}