{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91048,"databundleVersionId":10710872,"sourceType":"competition"},{"sourceId":10479950,"sourceType":"datasetVersion","datasetId":6489279}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#INITIAL XGBOOST MODEL (ATTEMPT 1)\nimport pandas as pd\nimport numpy as np\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import roc_auc_score, classification_report\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\n\n# Log function to track progress\ndef log(message):\n    print(f\"[LOG]: {message}\")\n\n# 1. Load Data\nlog(\"Loading data...\")\ndevelopment_data_path = '/kaggle/input/convolve/Dev_data_to_be_shared.csv'\nvalidation_data_path = '/kaggle/input/convolve/validation_data_to_be_shared.csv'\n\ndev_data = pd.read_csv(development_data_path)\nvalidation_data = pd.read_csv(validation_data_path)\n\nlog(\"Data loaded successfully.\")\n\n# Ensure consistent column names between datasets\ncommon_columns = dev_data.columns.intersection(validation_data.columns)\n\n# 2. Data Preprocessing\nlog(\"Starting data preprocessing...\")\n# Separate features and target\ntarget_column = 'bad_flag'\nX = dev_data[common_columns.drop('bad_flag', errors='ignore')]\ny = dev_data[target_column]\n\n# Validation data alignment\nX_val = validation_data[common_columns.drop('bad_flag', errors='ignore')]\n\n# Impute missing values\nlog(\"Imputing missing values...\")\nimputer = SimpleImputer(strategy='mean')\nX_imputed = imputer.fit_transform(X)\nX_val_imputed = imputer.transform(X_val)\n\n# Scale features\nlog(\"Scaling features...\")\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_imputed)\nX_val_scaled = scaler.transform(X_val_imputed)\n\n# 3. Train-Test Split\nlog(\"Splitting data into training and testing sets...\")\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n\n# 4. Model Training\nlog(\"Initializing XGBoost model with GPU support...\")\nmodel = XGBClassifier(\n    tree_method='gpu_hist',\n    predictor='gpu_predictor',\n    use_label_encoder=False,\n    random_state=42\n)\n\nlog(\"Training the model...\")\nmodel.set_params(eval_metric='auc')  # Avoid duplication error\nmodel.fit(\n    X_train, y_train,\n    eval_set=[(X_test, y_test)],\n    verbose=True\n)\n\n# 5. Evaluate the Model\nlog(\"Evaluating the model...\")\npreds = model.predict_proba(X_test)[:, 1]\nauc_score = roc_auc_score(y_test, preds)\nlog(f\"AUC-ROC Score: {auc_score}\")\nprint(classification_report(y_test, (preds > 0.5).astype(int)))\n\n# 6. Predict on Validation Data\nlog(\"Predicting probabilities for validation data...\")\nval_preds = model.predict_proba(X_val_scaled)[:, 1]\nvalidation_results = pd.DataFrame({\n    'account_number': validation_data['account_number'],\n    'predicted_probability': val_preds\n})\n\n# 7. Save Results\noutput_path = '/kaggle/working/validation_predictions.csv'\nvalidation_results.to_csv(output_path, index=False)\nlog(f\"Validation predictions saved to {output_path}\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:22:55.925915Z","iopub.execute_input":"2025-01-15T19:22:55.926196Z","iopub.status.idle":"2025-01-15T19:23:23.893874Z","shell.execute_reply.started":"2025-01-15T19:22:55.926173Z","shell.execute_reply":"2025-01-15T19:23:23.89309Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#LightGBM with SMOTE (Attempt 2)\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score, classification_report\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import SMOTE\n\n# Log function to track progress\ndef log(message):\n    print(f\"[LOG]: {message}\")\n\n# 1. Load Data\nlog(\"Loading data...\")\ndevelopment_data_path = '/kaggle/input/convolve/Dev_data_to_be_shared.csv'\nvalidation_data_path = '/kaggle/input/convolve/validation_data_to_be_shared.csv'\n\ndev_data = pd.read_csv(development_data_path)\nvalidation_data = pd.read_csv(validation_data_path)\n\nlog(\"Data loaded successfully.\")\n\n# Ensure consistent column names between datasets\ncommon_columns = dev_data.columns.intersection(validation_data.columns)\n\n# 2. Data Preprocessing\nlog(\"Starting data preprocessing...\")\n# Separate features and target\ntarget_column = 'bad_flag'\nX = dev_data[common_columns.drop('bad_flag', errors='ignore')]\ny = dev_data[target_column]\n\n# Validation data alignment\nX_val = validation_data[common_columns.drop('bad_flag', errors='ignore')]\n\n# Impute missing values\nlog(\"Imputing missing values...\")\nimputer = SimpleImputer(strategy='mean')\nX_imputed = imputer.fit_transform(X)\nX_val_imputed = imputer.transform(X_val)\n\n# Scale features\nlog(\"Scaling features...\")\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_imputed)\nX_val_scaled = scaler.transform(X_val_imputed)\n\n# 3. Train-Test Split\nlog(\"Splitting data into training and testing sets...\")\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42, stratify=y)\n\n# 4. Handle Class Imbalance with SMOTE\nlog(\"Applying SMOTE to handle class imbalance...\")\nsmote = SMOTE(random_state=42)\nX_train_resampled, y_train_resampled = smote.fit_resample(X_train, y_train)\n\n# 5. Model Training with LightGBM\nlog(\"Initializing LightGBM model...\")\nlgb_train = lgb.Dataset(X_train_resampled, label=y_train_resampled)\nlgb_test = lgb.Dataset(X_test, label=y_test, reference=lgb_train)\n\nparams = {\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"boosting_type\": \"gbdt\",\n    \"learning_rate\": 0.05,\n    \"num_leaves\": 31,\n    \"feature_fraction\": 0.8,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"verbose\": -1,\n    \"device\": \"gpu\",  # Use GPU for faster training\n}\n\nlog(\"Training LightGBM model...\")\nmodel = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=[lgb_train, lgb_test],\n    num_boost_round=1000,\n    callbacks=[lgb.early_stopping(stopping_rounds=50), lgb.log_evaluation(period=100)]\n)\n\n\n# 6. Evaluate the Model\nlog(\"Evaluating the model...\")\ny_test_preds = model.predict(X_test)\nauc_score = roc_auc_score(y_test, y_test_preds)\nlog(f\"AUC-ROC Score: {auc_score}\")\nprint(classification_report(y_test, (y_test_preds >= 0.5).astype(int)))\n\n# 7. Predict on Validation Data\nlog(\"Predicting probabilities for validation data...\")\nval_preds = model.predict(X_val_scaled)\nvalidation_results = pd.DataFrame({\n    'account_number': validation_data['account_number'],\n    'predicted_probability': val_preds\n})\n\n# 8. Save Results\noutput_path = '/kaggle/working/validation_predictions.csv'\nvalidation_results.to_csv(output_path, index=False)\nlog(f\"Validation predictions saved to {output_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:38:04.842266Z","iopub.execute_input":"2025-01-15T19:38:04.842556Z","iopub.status.idle":"2025-01-15T19:39:15.868764Z","shell.execute_reply.started":"2025-01-15T19:38:04.842533Z","shell.execute_reply":"2025-01-15T19:39:15.86798Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#\"FINAL SUBMISSION\" SMOTE+LIGHTGBM BASED MODEL (ATTEMPT 3)\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score, classification_report, precision_recall_curve\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import SMOTE, ADASYN\n\n# Log function to track progress\ndef log(message):\n    print(f\"[LOG]: {message}\")\n\n# 1. Load Data\nlog(\"Loading data...\")\ndevelopment_data_path = '/kaggle/input/convolve/Dev_data_to_be_shared.csv'\nvalidation_data_path = '/kaggle/input/convolve/validation_data_to_be_shared.csv'\n\ndev_data = pd.read_csv(development_data_path)\nvalidation_data = pd.read_csv(validation_data_path)\n\nlog(\"Data loaded successfully.\")\n\n# Ensure consistent column names between datasets\ncommon_columns = dev_data.columns.intersection(validation_data.columns)\n\n# 2. Data Preprocessing\nlog(\"Starting data preprocessing...\")\n# Separate features and target\ntarget_column = 'bad_flag'\nX = dev_data[common_columns.drop('bad_flag', errors='ignore')]\ny = dev_data[target_column]\n\n# Validation data alignment\nX_val = validation_data[common_columns.drop('bad_flag', errors='ignore')]\n\n# Impute missing values\nlog(\"Imputing missing values...\")\nimputer = SimpleImputer(strategy='mean')\nX_imputed = imputer.fit_transform(X)\nX_val_imputed = imputer.transform(X_val)\n\n# Scale features\nlog(\"Scaling features...\")\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_imputed)\nX_val_scaled = scaler.transform(X_val_imputed)\n\n# 3. Train-Test Split\nlog(\"Splitting data into training and testing sets...\")\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42, stratify=y)\n\n# 4. Handle Class Imbalance with Advanced Oversampling\nlog(\"Applying ADASYN to handle class imbalance...\")\nadasyn = ADASYN(random_state=42)\nX_train_resampled, y_train_resampled = adasyn.fit_resample(X_train, y_train)\n\n# 5. Model Training with LightGBM\nlog(\"Initializing LightGBM model...\")\nlgb_train = lgb.Dataset(X_train_resampled, label=y_train_resampled)\nlgb_test = lgb.Dataset(X_test, label=y_test, reference=lgb_train)\n\nparams = {\n    'objective': 'binary',\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'learning_rate': 0.01,\n    'num_leaves': 31,\n    'min_data_in_leaf': 50,  # Prevent overfitting\n    'scale_pos_weight': len(y_train[y_train == 0]) / len(y_train[y_train == 1]),  # Balance weights\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'device': 'gpu',  # Use GPU for faster training\n}\n\nlog(\"Training LightGBM model...\")\nmodel = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=[lgb_train, lgb_test],\n    num_boost_round=1000,\n    callbacks=[lgb.early_stopping(stopping_rounds=50), lgb.log_evaluation(period=100)]\n)\n\n# 6. Evaluate the Model with AUC and Optimized Threshold\nlog(\"Evaluating the model...\")\ny_test_probs = model.predict(X_test)\nauc_score = roc_auc_score(y_test, y_test_probs)\nlog(f\"AUC-ROC Score: {auc_score}\")\n\n# Optimize the decision threshold using Precision-Recall curve\nlog(\"Optimizing decision threshold...\")\nprecision, recall, thresholds = precision_recall_curve(y_test, y_test_probs)\nf1_scores = 2 * (precision * recall) / (precision + recall + 1e-6)  # Avoid divide-by-zero errors\noptimal_idx = np.argmax(f1_scores)\noptimal_threshold = thresholds[optimal_idx]\nlog(f\"Optimal Threshold: {optimal_threshold}\")\n\n# Evaluate with optimized threshold\ny_test_preds = (y_test_probs >= optimal_threshold).astype(int)\nprint(classification_report(y_test, y_test_preds))\n\n# 7. Predict on Validation Data with Optimized Threshold\nlog(\"Predicting probabilities for validation data...\")\nval_probs = model.predict(X_val_scaled)\nval_preds = (val_probs >= optimal_threshold).astype(int)\nvalidation_results = pd.DataFrame({\n    'account_number': validation_data['account_number'],\n    'predicted_probability': val_probs,\n    'predicted_class': val_preds\n})\n\n# 8. Save Results\noutput_path = '/kaggle/working/validation_predictions.csv'\nvalidation_results.to_csv(output_path, index=False)\nlog(f\"Validation predictions saved to {output_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:44:30.829732Z","iopub.execute_input":"2025-01-15T19:44:30.83004Z","iopub.status.idle":"2025-01-15T19:46:58.962436Z","shell.execute_reply.started":"2025-01-15T19:44:30.830016Z","shell.execute_reply":"2025-01-15T19:46:58.961631Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#ATTEMPT ON IMPROVEMENT ON BEST MODEL CATBOOST (ATTEMPT 4)\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom sklearn.metrics import roc_auc_score, classification_report, precision_recall_curve, average_precision_score\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import BorderlineSMOTE\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.combine import SMOTEENN\n\n# Log function to track progress\ndef log(message):\n    print(f\"[LOG]: {message}\")\n\n# 1. Load Data\nlog(\"Loading data...\")\ndevelopment_data_path = '/kaggle/input/convolve/Dev_data_to_be_shared.csv'\nvalidation_data_path = '/kaggle/input/convolve/validation_data_to_be_shared.csv'\n\ndev_data = pd.read_csv(development_data_path)\nvalidation_data = pd.read_csv(validation_data_path)\n\nlog(\"Data loaded successfully.\")\n\n# Ensure consistent column names between datasets\ncommon_columns = dev_data.columns.intersection(validation_data.columns)\n\n# 2. Data Preprocessing\nlog(\"Starting data preprocessing...\")\n# Separate features and target\ntarget_column = 'bad_flag'\nX = dev_data[common_columns.drop('bad_flag', errors='ignore')]\ny = dev_data[target_column]\n\n# Validation data alignment\nX_val = validation_data[common_columns.drop('bad_flag', errors='ignore')]\n\n# Impute missing values\nlog(\"Imputing missing values...\")\nimputer = SimpleImputer(strategy='mean')\nX_imputed = imputer.fit_transform(X)\nX_val_imputed = imputer.transform(X_val)\n\n# Scale features\nlog(\"Scaling features...\")\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_imputed)\nX_val_scaled = scaler.transform(X_val_imputed)\n\n# 3. Train-Test Split\nlog(\"Splitting data into training and testing sets...\")\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42, stratify=y)\n\n# 4. Handle Class Imbalance with Separate Oversampling and Undersampling\nlog(\"Applying Borderline-SMOTE followed by undersampling...\")\n# Step 1: Oversample the minority class using Borderline-SMOTE\nsmote = BorderlineSMOTE(random_state=42, kind='borderline-2')\nX_train_oversampled, y_train_oversampled = smote.fit_resample(X_train, y_train)\n\n# Step 2: Undersample the majority class using RandomUnderSampler\nunder_sampler = RandomUnderSampler(random_state=42)\nX_train_resampled, y_train_resampled = under_sampler.fit_resample(X_train_oversampled, y_train_oversampled)\n\n# 5. Model Training with LightGBM\nlog(\"Initializing LightGBM model...\")\nlgb_train = lgb.Dataset(X_train_resampled, label=y_train_resampled)\nlgb_test = lgb.Dataset(X_test, label=y_test, reference=lgb_train)\n\nparams = {\n    'objective': 'binary',\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'learning_rate': 0.005,\n    'num_leaves': 15,  # Lower complexity\n    'max_depth': 7,\n    'min_data_in_leaf': 100,  # Prevent overfitting\n    'scale_pos_weight': len(y_train[y_train == 0]) / len(y_train[y_train == 1]),  # Balance weights\n    'feature_fraction': 0.7,\n    'bagging_fraction': 0.7,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'device': 'gpu',  # Use GPU for faster training\n}\n\nlog(\"Training LightGBM model...\")\nmodel = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=[lgb_train, lgb_test],\n    num_boost_round=2000,\n    callbacks=[lgb.early_stopping(stopping_rounds=50), lgb.log_evaluation(period=100)]\n)\n\n# 6. Evaluate the Model\nlog(\"Evaluating the model...\")\ny_test_probs = model.predict(X_test)\nauc_score = roc_auc_score(y_test, y_test_probs)\npr_auc_score = average_precision_score(y_test, y_test_probs)\nlog(f\"AUC-ROC Score: {auc_score}\")\nlog(f\"Precision-Recall AUC Score: {pr_auc_score}\")\n\n# Optimize the decision threshold using Precision-Recall curve\nlog(\"Optimizing decision threshold...\")\nprecision, recall, thresholds = precision_recall_curve(y_test, y_test_probs)\nf1_scores = 2 * (precision * recall) / (precision + recall + 1e-6)  # Avoid divide-by-zero errors\noptimal_idx = np.argmax(f1_scores)\noptimal_threshold = thresholds[optimal_idx]\nlog(f\"Optimal Threshold: {optimal_threshold}\")\n\n# Evaluate with optimized threshold\ny_test_preds = (y_test_probs >= optimal_threshold).astype(int)\nprint(classification_report(y_test, y_test_preds))\n\n# 7. Predict on Validation Data with Optimized Threshold\nlog(\"Predicting probabilities for validation data...\")\nval_probs = model.predict(X_val_scaled)\nval_preds = (val_probs >= optimal_threshold).astype(int)\nvalidation_results = pd.DataFrame({\n    'account_number': validation_data['account_number'],\n    'predicted_probability': val_probs,\n    'predicted_class': val_preds\n})\n\n# 8. Save Results\noutput_path = '/kaggle/working/validation_predictions.csv'\nvalidation_results.to_csv(output_path, index=False)\nlog(f\"Validation predictions saved to {output_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T19:53:18.410018Z","iopub.execute_input":"2025-01-15T19:53:18.410324Z","iopub.status.idle":"2025-01-15T19:55:51.491833Z","shell.execute_reply.started":"2025-01-15T19:53:18.4103Z","shell.execute_reply":"2025-01-15T19:55:51.490939Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#ATTEMPTED ADASYN+RANDOM FOREST ENSEMBLE (ATTEMPT 5)\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score, classification_report, precision_recall_curve\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import ADASYN, SMOTE\nfrom imblearn.combine import SMOTEENN \nfrom sklearn.ensemble import RandomForestClassifier\n\n# Log function to track progress\ndef log(message):\n    print(f\"[LOG]: {message}\")\n\n# 1. Load Data\nlog(\"Loading data...\")\ndevelopment_data_path = '/kaggle/input/convolve/Dev_data_to_be_shared.csv'\nvalidation_data_path = '/kaggle/input/convolve/validation_data_to_be_shared.csv'\n\ndev_data = pd.read_csv(development_data_path)\nvalidation_data = pd.read_csv(validation_data_path)\n\nlog(\"Data loaded successfully.\")\n\n# Ensure consistent column names between datasets\ncommon_columns = dev_data.columns.intersection(validation_data.columns)\n\n# 2. Data Preprocessing\nlog(\"Starting data preprocessing...\")\n# Separate features and target\ntarget_column = 'bad_flag'\nX = dev_data[common_columns.drop('bad_flag', errors='ignore')]\ny = dev_data[target_column]\n\n# Validation data alignment\nX_val = validation_data[common_columns.drop('bad_flag', errors='ignore')]\n\n# Impute missing values\nlog(\"Imputing missing values...\")\nimputer = SimpleImputer(strategy='mean')\nX_imputed = imputer.fit_transform(X)\nX_val_imputed = imputer.transform(X_val)\n\n# Scale features\nlog(\"Scaling features...\")\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_imputed)\nX_val_scaled = scaler.transform(X_val_imputed)\n\n# 3. Train-Test Split\nlog(\"Splitting data into training and testing sets...\")\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42, stratify=y)\n\n# 4. Handle Class Imbalance with Advanced Oversampling\nlog(\"Applying ADASYN to handle class imbalance...\")\nadasyn = ADASYN(random_state=42)\nX_train_resampled, y_train_resampled = adasyn.fit_resample(X_train, y_train)\n\n# Alternatively, use SMOTE+Tomek Links for class imbalance handling\n# smote = SMOTE(random_state=42)\n# tomek_oversample = SMOTEENN(sampling_strategy='auto', smote=smote)\n# X_train_resampled, y_train_resampled = tomek_oversample.fit_resample(X_train, y_train)\n\n# 5. Model Training with LightGBM\nlog(\"Initializing LightGBM model...\")\nlgb_train = lgb.Dataset(X_train_resampled, label=y_train_resampled)\nlgb_test = lgb.Dataset(X_test, label=y_test, reference=lgb_train)\n\nparams = {\n    'objective': 'binary',\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'learning_rate': 0.01,\n    'num_leaves': 31,\n    'min_data_in_leaf': 50,  # Prevent overfitting\n    'scale_pos_weight': len(y_train[y_train == 0]) / len(y_train[y_train == 1]),  # Balance weights\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'device': 'gpu',  # Use GPU for faster training\n}\n\nlog(\"Training LightGBM model...\")\nmodel = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=[lgb_train, lgb_test],\n    num_boost_round=1000,\n    callbacks=[lgb.early_stopping(stopping_rounds=50), lgb.log_evaluation(period=100)]\n)\n\n# 6. Evaluate the Model with AUC and Optimized Threshold\nlog(\"Evaluating the model...\")\ny_test_probs = model.predict(X_test)\nauc_score = roc_auc_score(y_test, y_test_probs)\nlog(f\"AUC-ROC Score: {auc_score}\")\n\n# Optimize the decision threshold using Precision-Recall curve\nlog(\"Optimizing decision threshold...\")\nprecision, recall, thresholds = precision_recall_curve(y_test, y_test_probs)\nf1_scores = 2 * (precision * recall) / (precision + recall + 1e-6)  # Avoid divide-by-zero errors\noptimal_idx = np.argmax(f1_scores)\noptimal_threshold = thresholds[optimal_idx]\nlog(f\"Optimal Threshold: {optimal_threshold}\")\n\n# Evaluate with optimized threshold\ny_test_preds = (y_test_probs >= optimal_threshold).astype(int)\nlog(classification_report(y_test, y_test_preds))\n\n# 7. Predict on Validation Data with Optimized Threshold\nlog(\"Predicting probabilities for validation data...\")\nval_probs = model.predict(X_val_scaled)\nval_preds = (val_probs >= optimal_threshold).astype(int)\nvalidation_results = pd.DataFrame({\n    'account_number': validation_data['account_number'],\n    'predicted_probability': val_probs,\n    'predicted_class': val_preds\n})\n\n# 8. Save Results\noutput_path = '/kaggle/working/validation_predictions.csv'\nvalidation_results.to_csv(output_path, index=False)\nlog(f\"Validation predictions saved to {output_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T20:02:25.149877Z","iopub.execute_input":"2025-01-15T20:02:25.150168Z","iopub.status.idle":"2025-01-15T20:04:54.727443Z","shell.execute_reply.started":"2025-01-15T20:02:25.150146Z","shell.execute_reply":"2025-01-15T20:04:54.726691Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#SMOTE+PCA+TOMEK LINKS ALGORITHM (ATTEMPT 6)\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.metrics import classification_report, precision_recall_curve, roc_auc_score\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom imblearn.over_sampling import SMOTE\nfrom imblearn.under_sampling import TomekLinks\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.decomposition import PCA\nfrom joblib import Parallel, delayed\n\n# Log function to track progress\ndef log(message):\n    print(f\"[LOG]: {message}\")\n\n# Load Data\nlog(\"Loading data...\")\ndevelopment_data_path = '/kaggle/input/convolve/Dev_data_to_be_shared.csv'\nvalidation_data_path = '/kaggle/input/convolve/validation_data_to_be_shared.csv'\n\ndev_data = pd.read_csv(development_data_path)\nvalidation_data = pd.read_csv(validation_data_path)\n\nlog(\"Data loaded successfully.\")\n\n# Ensure consistent column names between datasets\ncommon_columns = dev_data.columns.intersection(validation_data.columns)\n\n# Data Preprocessing\nlog(\"Starting data preprocessing...\")\ntarget_column = 'bad_flag'\nX = dev_data[common_columns.drop('bad_flag', errors='ignore')]\ny = dev_data[target_column]\n\nX_val = validation_data[common_columns.drop('bad_flag', errors='ignore')]\n\n# Impute missing values\nlog(\"Imputing missing values...\")\nimputer = SimpleImputer(strategy='mean')\nX_imputed = imputer.fit_transform(X)\nX_val_imputed = imputer.transform(X_val)\n\n# Scale features\nlog(\"Scaling features...\")\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_imputed)\nX_val_scaled = scaler.transform(X_val_imputed)\n\n# Apply PCA for dimensionality reduction\nlog(\"Applying PCA for dimensionality reduction...\")\npca = PCA(n_components=0.95)  # Retain 95% variance\nX_scaled_pca = pca.fit_transform(X_scaled)\nX_val_scaled_pca = pca.transform(X_val_scaled)\n\n# Train-Test Split\nlog(\"Splitting data into training and testing sets...\")\nX_train, X_test, y_train, y_test = train_test_split(X_scaled_pca, y, test_size=0.2, random_state=42, stratify=y)\n\n# Apply SMOTE with optimized nearest neighbor search\nlog(\"Applying SMOTE for optimized nearest neighbor search...\")\nsmote = SMOTE(sampling_strategy='auto', k_neighbors=5, n_jobs=-1, random_state=42)\nX_train_resampled, y_train_resampled = smote.fit_resample(X_train, y_train)\n\n# Apply Tomek Links after SMOTE with Parallelization\nlog(\"Applying Tomek Links to remove noisy points...\")\ntomek = TomekLinks(n_jobs=-1)\nX_train_resampled, y_train_resampled = tomek.fit_resample(X_train_resampled, y_train_resampled)\n\n# Hyperparameter Tuning with Randomized Search\nlog(\"Tuning LightGBM hyperparameters...\")\nparam_dist = {\n    'num_leaves': [31, 63, 127],\n    'min_data_in_leaf': [20, 50],\n    'scale_pos_weight': [len(y_train[y_train == 0]) / len(y_train[y_train == 1]), 1],\n    'learning_rate': [0.01, 0.05],\n    'boosting_type': ['gbdt'],\n    'gpu_use_dp': [True]  # Enable double precision mode for GPU\n}\n\nlgb_model = lgb.LGBMClassifier(objective='binary', metric='auc', device='gpu')\n\n# Randomized Search for Hyperparameters\nrandom_search = RandomizedSearchCV(lgb_model, param_dist, n_iter=20, cv=3, n_jobs=-1, verbose=2, random_state=42)\n\n# Perform Randomized Search without early_stopping_rounds in fit\nrandom_search.fit(X_train_resampled, y_train_resampled)\n\n# Get the best model from RandomizedSearchCV\nbest_lgb_model = random_search.best_estimator_\n\n# Train the best model with early stopping manually\nlog(\"Training best model with early stopping...\")\nbest_lgb_model.fit(X_train_resampled, y_train_resampled, eval_set=[(X_test, y_test)], \n                   eval_metric='auc', early_stopping_rounds=50, verbose=10)\n\n# Evaluate the Model with AUC and Optimized Threshold\nlog(\"Evaluating the model...\")\ny_test_probs = best_lgb_model.predict_proba(X_test)[:, 1]\nauc_score = roc_auc_score(y_test, y_test_probs)\nlog(f\"AUC-ROC Score: {auc_score}\")\n\n# Optimize the decision threshold using Precision-Recall curve\nlog(\"Optimizing decision threshold...\")\nprecision, recall, thresholds = precision_recall_curve(y_test, y_test_probs)\nf1_scores = 2 * (precision * recall) / (precision + recall + 1e-6)  # Avoid divide-by-zero errors\noptimal_idx = np.argmax(f1_scores)\noptimal_threshold = thresholds[optimal_idx]\nlog(f\"Optimal Threshold: {optimal_threshold}\")\n\n# Evaluate with optimized threshold\ny_test_preds = (y_test_probs >= optimal_threshold).astype(int)\nlog(\"Classification Report (LightGBM):\")\nprint(classification_report(y_test, y_test_preds))\n\n# Ensemble Method: Random Forest\nlog(\"Training Random Forest model...\")\nrf_model = RandomForestClassifier(n_estimators=100, random_state=42, class_weight='balanced')\nrf_model.fit(X_train_resampled, y_train_resampled)\n\n# Evaluate Random Forest Model\nlog(\"Evaluating Random Forest model...\")\ny_rf_probs = rf_model.predict_proba(X_test)[:, 1]\nrf_auc_score = roc_auc_score(y_test, y_rf_probs)\nlog(f\"Random Forest AUC-ROC Score: {rf_auc_score}\")\n\n# Final Evaluation and Submission Preparation\nlog(\"Predicting on validation data with optimized threshold...\")\nval_probs = best_lgb_model.predict_proba(X_val_scaled_pca)[:, 1]\nval_preds = (val_probs >= optimal_threshold).astype(int)\nvalidation_results = pd.DataFrame({\n    'account_number': validation_data['account_number'],\n    'predicted_probability': val_probs,\n    'predicted_class': val_preds\n})\n\n# Save Results\noutput_path = '/kaggle/working/validation_predictions.csv'\nvalidation_results.to_csv(output_path, index=False)\nlog(f\"Validation predictions saved to {output_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-15T20:52:30.582974Z","iopub.execute_input":"2025-01-15T20:52:30.583315Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null}]}