{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import packages","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\nimport os\nimport gc\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\n\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:38:42.078494Z","iopub.execute_input":"2024-06-27T15:38:42.079442Z","iopub.status.idle":"2024-06-27T15:38:45.160438Z","shell.execute_reply.started":"2024-06-27T15:38:42.079404Z","shell.execute_reply":"2024-06-27T15:38:45.159161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset\n\nWe use `polars` instead of `pandas` for better efficiency in processing huge dataset.","metadata":{}},{"cell_type":"code","source":"# Define dtypes\ntrain_schema = {\n    'id': pl.UInt64,\n    'click': pl.Int8,\n    'hour': pl.String,\n    'C1': pl.Int16,\n    'banner_pos': pl.Int8,\n    'site_id': pl.Categorical,\n    'site_domain': pl.Categorical,\n    'site_category': pl.Categorical,\n    'app_id': pl.Categorical,\n    'app_domain': pl.Categorical,\n    'app_category': pl.Categorical,\n    'device_id': pl.Categorical,\n    'device_ip': pl.Categorical,\n    'device_model': pl.Categorical,\n    'device_type': pl.Int8,\n    'device_conn_type': pl.Int8,\n    'C14': pl.Int16,\n    'C15': pl.Int16,\n    'C16': pl.Int16,\n    'C17': pl.Int16,\n    'C18': pl.Int16,\n    'C19': pl.Int16,\n    'C20': pl.Int32,\n    'C21': pl.Int16\n}\n\ntest_schema = {\n    'id': pl.UInt64,\n    'hour': pl.String,\n    'C1': pl.Int16,\n    'banner_pos': pl.Int8,\n    'site_id': pl.Categorical,\n    'site_domain': pl.Categorical,\n    'site_category': pl.Categorical,\n    'app_id': pl.Categorical,\n    'app_domain': pl.Categorical,\n    'app_category': pl.Categorical,\n    'device_id': pl.Categorical,\n    'device_ip': pl.Categorical,\n    'device_model': pl.Categorical,\n    'device_type': pl.Int8,\n    'device_conn_type': pl.Int8,\n    'C14': pl.Int16,\n    'C15': pl.Int16,\n    'C16': pl.Int16,\n    'C17': pl.Int16,\n    'C18': pl.Int16,\n    'C19': pl.Int16,\n    'C20': pl.Int32,\n    'C21': pl.Int16\n}\n\ntrain_df = pl.read_csv('../input/avazu-ctr-prediction/train.gz', schema=train_schema)\ntest_df = pl.read_csv('../input/avazu-ctr-prediction/test.gz', schema=test_schema)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:38:45.162549Z","iopub.execute_input":"2024-06-27T15:38:45.163148Z","iopub.status.idle":"2024-06-27T15:39:59.443957Z","shell.execute_reply.started":"2024-06-27T15:38:45.163117Z","shell.execute_reply":"2024-06-27T15:39:59.442711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"========== Size of Datasets ==========\")\nprint(f\"Train Dataset: {train_df.shape}\")\nprint(f\"Test Dataset: {test_df.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:13.263253Z","iopub.execute_input":"2024-06-27T12:29:13.263716Z","iopub.status.idle":"2024-06-27T12:29:13.272092Z","shell.execute_reply.started":"2024-06-27T12:29:13.263678Z","shell.execute_reply":"2024-06-27T12:29:13.270761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Random sampling from the training dataset\n\nTo increase the training efficiency, we sample the training data randomly from the original training dataset.","metadata":{}},{"cell_type":"code","source":"n_samples = 5000000\nselected_idx = sorted(np.random.choice(np.array([i for i in range(train_df.shape[0])]), size=n_samples, replace=False))\ntrain_df = train_df[selected_idx]","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:39:59.446003Z","iopub.execute_input":"2024-06-27T15:39:59.446323Z","iopub.status.idle":"2024-06-27T15:40:18.816100Z","shell.execute_reply.started":"2024-06-27T15:39:59.446296Z","shell.execute_reply":"2024-06-27T15:40:18.814865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observation\n\nCheck values in the dataset.","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:29.050768Z","iopub.execute_input":"2024-06-27T12:29:29.051204Z","iopub.status.idle":"2024-06-27T12:29:29.072944Z","shell.execute_reply.started":"2024-06-27T12:29:29.051168Z","shell.execute_reply":"2024-06-27T12:29:29.071582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imbalanced data\n\nAs you can see in the figure below, there exists imbalanced data of the target value.\n\nWe will show our techniques later to conquer this problem.","metadata":{}},{"cell_type":"code","source":"train_df['click'].value_counts().to_pandas().plot(x='click', y='count', kind='bar')","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:29.074578Z","iopub.execute_input":"2024-06-27T12:29:29.075120Z","iopub.status.idle":"2024-06-27T12:29:29.765058Z","shell.execute_reply.started":"2024-06-27T12:29:29.075072Z","shell.execute_reply":"2024-06-27T12:29:29.763371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\n## Add features\n\nThe column `hour` is represented as the datetime of data collected.\n\nWe can add two features based on this column, named `weekday` and `hour_time`, representing the weekday and the hour of data collected respectively.","metadata":{}},{"cell_type":"code","source":"train_df = train_df.with_columns((pl.col('hour') + '00').str.to_datetime(format=\"%y%m%d%H%M\"))\ntrain_df = train_df.with_columns(train_df['hour'].dt.weekday().alias('weekday'))\ntrain_df = train_df.with_columns(train_df['hour'].dt.hour().alias('hour_time'))\n\ntest_df = test_df.with_columns((pl.col('hour') + '00').str.to_datetime(format=\"%y%m%d%H%M\"))\ntest_df = test_df.with_columns(test_df['hour'].dt.weekday().alias('weekday'))\ntest_df = test_df.with_columns(test_df['hour'].dt.hour().alias('hour_time'))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:18.817488Z","iopub.execute_input":"2024-06-27T15:40:18.817813Z","iopub.status.idle":"2024-06-27T15:40:20.137907Z","shell.execute_reply.started":"2024-06-27T15:40:18.817785Z","shell.execute_reply":"2024-06-27T15:40:20.136946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection\n\nNow we have a lot of features. To avoid too much noise introduced in our model, we should only pick up important features. \n\n__Note__: The following columns will be skipped:\n1. `id`: It is just used for serial numbers, not for any special meaning.\n2. `hour`: We replaced it with columns `weekday` and `hour_time` to represent the time of data collected.\n3. `click`: This is the target column, so we must drop it in the training process.","metadata":{}},{"cell_type":"code","source":"numerical_dtypes = [pl.Int8, pl.Int16, pl.Int32, pl.Int64]\n\n# We should distinguish numerical and categorical features.\nnumerical_feats = []\ncategorical_feats = []\n\nskip_cols = ['id', 'hour', 'click']\nfor col in train_df.columns:\n    if (col in skip_cols):\n        continue\n        \n    if (train_df[col].dtype in numerical_dtypes):\n        numerical_feats.append(col)\n    else:\n        categorical_feats.append(col)\n        \nprint(f\"Numerical Features: {numerical_feats}\", '\\n')\nprint(f\"Categorical Features: {categorical_feats}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:20.140327Z","iopub.execute_input":"2024-06-27T15:40:20.140677Z","iopub.status.idle":"2024-06-27T15:40:20.149138Z","shell.execute_reply.started":"2024-06-27T15:40:20.140648Z","shell.execute_reply":"2024-06-27T15:40:20.148015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Numerical features\n### Drop features for excessive missing/abnormal data\n\nIf you observe `train_df` closely, you might find the column `C20` exists value \"-1\".\n\nIn general, \"-1\" is regarded as abnormal or missing data. If there are too many abnormal/missing data in one feature, it might downperform the predictive power of the model.\n\nAs you can see in the following cell, \"-1\" takes a large percentage (almost 47%) in the column `C20`. It indicates that removing the column `C20` might be better.","metadata":{}},{"cell_type":"code","source":"C20_counts = train_df['C20'].value_counts().sort('count', descending=True)\nC20_counts.with_columns((pl.col('count')/pl.sum('count')*100).round(2).alias('percentage(%)'))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:20.151145Z","iopub.execute_input":"2024-06-27T15:40:20.151609Z","iopub.status.idle":"2024-06-27T15:40:20.311779Z","shell.execute_reply.started":"2024-06-27T15:40:20.151571Z","shell.execute_reply":"2024-06-27T15:40:20.310710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Drop high-correlated features\n\nThe next step is to eliminate high-correlated features. If we do not conduct this step and directly build the model, it would cause a collinearity problem, leading to the model being underperformance.","metadata":{}},{"cell_type":"code","source":"corr_arr = train_df[numerical_feats].corr().to_numpy()\nfor idxloc in np.argwhere(corr_arr>0.7):\n    i, j = idxloc\n    if (numerical_feats[i] != numerical_feats[j]) and (i < j):\n        print(f\"`{numerical_feats[i]}` & `{numerical_feats[j]}` are highly correlated with a corrcoeff = {round(corr_arr[i][j], 2)}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:20.313296Z","iopub.execute_input":"2024-06-27T15:40:20.313834Z","iopub.status.idle":"2024-06-27T15:40:20.970253Z","shell.execute_reply.started":"2024-06-27T15:40:20.313789Z","shell.execute_reply":"2024-06-27T15:40:20.969180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visulization of Pearson Corr\nplt.figure(figsize=((15,10)))\nsns.heatmap(corr_arr,\n            xticklabels=numerical_feats, yticklabels=numerical_feats,\n            cmap='Blues', annot=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:31.744224Z","iopub.execute_input":"2024-06-27T12:29:31.745126Z","iopub.status.idle":"2024-06-27T12:29:32.840704Z","shell.execute_reply.started":"2024-06-27T12:29:31.745080Z","shell.execute_reply":"2024-06-27T12:29:32.839499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Eliminate redundant features\nremove_feats = ['C20', 'C1', 'C14']\nfor feat in remove_feats:\n    numerical_feats.remove(feat)\nprint(f\"Selected Numerical Features: {numerical_feats}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:20.971736Z","iopub.execute_input":"2024-06-27T15:40:20.972183Z","iopub.status.idle":"2024-06-27T15:40:20.978364Z","shell.execute_reply.started":"2024-06-27T15:40:20.972146Z","shell.execute_reply":"2024-06-27T15:40:20.977265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical features\n\nWhen it comes to independence between subgroups under a feature, we can conduct a Chi-Square test for independence. However, according to the cell below, it is naive to conduct a Chi-Square test for independence in these categorical features because each feature has too many subgroups. Hence, we retain all categorical features at first.\n\nAlso, there is an issue we have not solved, named __categorical feature encoding__.\n\nThere are several encoding approaches, such as one-hot encoding, frequency encoding, target encoding, etc. In this notebook, we will adopt __LightGBM__, which has great native categorical feature support. Therefore, we do not have to encode categorical features in advance.","metadata":{}},{"cell_type":"code","source":"for feat in categorical_feats:\n    print(f\"Categorical feature `{feat}` has {train_df[feat].n_unique()} groups.\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:32.851326Z","iopub.execute_input":"2024-06-27T12:29:32.851676Z","iopub.status.idle":"2024-06-27T12:29:33.612657Z","shell.execute_reply.started":"2024-06-27T12:29:32.851648Z","shell.execute_reply":"2024-06-27T12:29:33.611447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:33.614041Z","iopub.execute_input":"2024-06-27T12:29:33.614382Z","iopub.status.idle":"2024-06-27T12:29:33.979157Z","shell.execute_reply.started":"2024-06-27T12:29:33.614354Z","shell.execute_reply":"2024-06-27T12:29:33.977814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training\n## Training Strategy: Cross Validation\n### K-Fold\nK-Fold cross-validation is a common technique in ML applications, which divides training datasets into $N$ chunks ($N$ = 5 or 10 in general). In each training subprocess, one of these chunks is regarded as validation data, and the others are used as training data. If all chunks of the training dataset have been regarded as validation data once, the whole training process is done.\n\n### Stratified K-Fold\nStratified K-Fold is a variant of K-Fold cross-validation that ensures the distribution of target classes in each chunk is representative of the overall dataset. This approach is useful for imbalanced datasets, leading to more robust and generalizable model evaluation.\n\n## LightGBM\nWe always have to encode categorical features properly before building models. There are many approaches for categorical feature encoding. However, to save time, we use LightGBM for its great native categorical feature support instead of encoding by ourselves. ","metadata":{}},{"cell_type":"code","source":"def load_chunk_dataset(X: pd.DataFrame, y: pd.Series, chunk_size=5000000):\n    \"\"\" To save RAM, we load the training dataset in chunks and train in chunks \"\"\"\n    for idx in range(0, len(X), chunk_size):\n        yield X[idx:idx+chunk_size], y[idx:idx+chunk_size]\n        \ndef model_eval(k: int, X: pd.DataFrame, y: pd.Series, decimal: int = 4) -> tuple:\n    y_pred = model.predict(X)\n    y_pred_probs = model.predict_proba(X)\n    \n    accuracy_score = model.score(X, y)\n    ce_loss = metrics.log_loss(y_true=y, y_pred=y_pred_probs[:,1])\n    auc_score = metrics.roc_auc_score(y_true=y, y_score=y_pred_probs[:,1])\n    cnf_matrix = metrics.confusion_matrix(y, y_pred)\n    \n    print(f\"========== Model Evaluation on Fold {k} ==========\")\n    print(f\"Accuracy: {round(accuracy_score, decimal)}\")\n    print(f\"Cross-Entropy Loss: {round(ce_loss, decimal)}\")\n    print(f\"AUC: {round(auc_score, decimal)}\")\n    print(\"================================================\", '\\n')\n    \n    return y_pred, y_pred_probs, accuracy_score, ce_loss, auc_score, cnf_matrix\n    \ndef draw_roc_curve(y_true: pd.Series, y_pred_probs: np.ndarray):\n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred_probs[:,1], drop_intermediate=False)\n    plt.figure(figsize=(6, 6))\n    plt.plot(fpr, tpr, label=f\"ROC curve (area = {auc:.2f})\")\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic Curve (ROC)')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \ndef confusion_matrix_info(cnf_matrix: np.ndarray) -> tuple:\n    TP = cnf_matrix[1][1]\n    TN = cnf_matrix[0][0]\n    FP = cnf_matrix[0][1]\n    FN = cnf_matrix[1][0]    \n    return TP, TN, FP, FN\n\ndef print_metrics_info(records_df: pd.DataFrame, metrics_list: list):\n    metrics_info = records_df.mean()\n    print('========== Evaluation Metrics ==========')\n    for metric in metrics_list:\n        print(f\"{metric} = {metrics_info.loc[metric]:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:20.979995Z","iopub.execute_input":"2024-06-27T15:40:20.980457Z","iopub.status.idle":"2024-06-27T15:40:21.197826Z","shell.execute_reply.started":"2024-06-27T15:40:20.980419Z","shell.execute_reply":"2024-06-27T15:40:21.196565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the whole training dataset into 10 chunks\nskf = StratifiedKFold(n_splits=10)\nselected_feats = numerical_feats + categorical_feats\nX = train_df[selected_feats].to_pandas()\ny = train_df['click'].to_pandas()\n\n# Define model and its params\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"learning_rate\": 0.03,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"device\": \"cpu\",\n}\nmodel = lgb.LGBMClassifier(**params)\nrecords = []\n\n# Start training\nfor k, (train_idx, valid_idx) in enumerate(skf.split(X, y)):\n    print(f\"Fold {k}: \")\n    X_train = X.loc[train_idx]\n    y_train = y.loc[train_idx]\n    \n    X_valid = X.loc[valid_idx]\n    y_valid = y.loc[valid_idx]\n    \n    # Convert to pandas dataframe due to better works between lightgbm's dataset & pandas dataframe\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        categorical_feature=categorical_feats,\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(100)]\n    )\n    \n    y_pred, y_pred_probs, accuracy_score, ce_loss, auc_score, cnf_matrix =\\\n        model_eval(k, X_valid, y_valid)\n    \n    TP, TN, FP, FN = confusion_matrix_info(cnf_matrix)\n    \n    # Save Evaluation Results\n    record_dict = {}\n    record_dict['Accuracy'] = accuracy_score\n    record_dict['Cross_Entropy_Loss'] = ce_loss\n    record_dict['AUC'] = auc_score\n    record_dict['TP'] = TP\n    record_dict['TN'] = TN\n    record_dict['FP'] = FP\n    record_dict['FN'] = FN\n    records.append(record_dict)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T12:29:34.000860Z","iopub.execute_input":"2024-06-27T12:29:34.001371Z","iopub.status.idle":"2024-06-27T13:55:21.387840Z","shell.execute_reply.started":"2024-06-27T12:29:34.001327Z","shell.execute_reply":"2024-06-27T13:55:21.385984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"records_df = pd.DataFrame(records)\nrecords_df['Precision'] = records_df.apply(lambda x: (x['TP']) / (x['TP']+x['FP']), axis=1)\nrecords_df['Recall'] = records_df.apply(lambda x: (x['TP']) / (x['TP']+x['FN']), axis=1)\nrecords_df","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:55:21.423640Z","iopub.execute_input":"2024-06-27T13:55:21.424023Z","iopub.status.idle":"2024-06-27T13:55:21.623088Z","shell.execute_reply.started":"2024-06-27T13:55:21.423994Z","shell.execute_reply":"2024-06-27T13:55:21.621960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics_list=['AUC', 'Cross_Entropy_Loss','Accuracy','Precision','Recall']\nprint_metrics_info(records_df, metrics_list)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:55:21.624813Z","iopub.execute_input":"2024-06-27T13:55:21.625254Z","iopub.status.idle":"2024-06-27T13:55:21.638841Z","shell.execute_reply.started":"2024-06-27T13:55:21.625219Z","shell.execute_reply":"2024-06-27T13:55:21.637705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance\n\nWe plot the importance score of each feature. According to the figure shown below, we reselect the features whose feature score is greater than 1000, then try for another training.","metadata":{}},{"cell_type":"code","source":"lgb.plot_importance(model, figsize=(10,6))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:55:21.640297Z","iopub.execute_input":"2024-06-27T13:55:21.640686Z","iopub.status.idle":"2024-06-27T13:55:22.129420Z","shell.execute_reply.started":"2024-06-27T13:55:21.640643Z","shell.execute_reply":"2024-06-27T13:55:22.128245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Again","metadata":{}},{"cell_type":"code","source":"numerical_feats_2 = ['C17', 'hour_time', 'C19', 'C21']\ncategorical_feats_2 = ['device_model', 'device_ip', 'site_id', 'app_id', 'site_domain']","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:21.199258Z","iopub.execute_input":"2024-06-27T15:40:21.199584Z","iopub.status.idle":"2024-06-27T15:40:21.211077Z","shell.execute_reply.started":"2024-06-27T15:40:21.199556Z","shell.execute_reply":"2024-06-27T15:40:21.209866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the whole training dataset into 10 chunks\nskf = StratifiedKFold(n_splits=10)\nreselected_feats = numerical_feats_2 + categorical_feats_2\nX = train_df[reselected_feats].to_pandas()\ny = train_df['click'].to_pandas()\n\n# Define model and its params\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"learning_rate\": 0.03,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"device\": \"cpu\",\n}\nmodel_2 = lgb.LGBMClassifier(**params)\nrecords_2 = []\n\n# Start training\nfor k, (train_idx, valid_idx) in enumerate(skf.split(X, y)):\n    print(f\"Fold {k}: \")\n    X_train = X.loc[train_idx]\n    y_train = y.loc[train_idx]\n    \n    X_valid = X.loc[valid_idx]\n    y_valid = y.loc[valid_idx]\n    \n    # Convert to pandas dataframe due to better works between lightgbm's dataset & pandas dataframe\n    model_2.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        categorical_feature=categorical_feats_2,\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(100)]\n    )\n    \n    y_pred, y_pred_probs, accuracy_score, ce_loss, auc_score, cnf_matrix =\\\n        model_eval(k, X_valid, y_valid)\n    \n    TP, TN, FP, FN = confusion_matrix_info(cnf_matrix)\n    \n    # Save Evaluation Results\n    record_dict = {}\n    record_dict['Accuracy'] = accuracy_score\n    record_dict['Cross_Entropy_Loss'] = ce_loss\n    record_dict['AUC'] = auc_score\n    record_dict['TP'] = TP\n    record_dict['TN'] = TN\n    record_dict['FP'] = FP\n    record_dict['FN'] = FN\n    records_2.append(record_dict)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T15:40:21.215027Z","iopub.execute_input":"2024-06-27T15:40:21.215415Z","iopub.status.idle":"2024-06-27T17:43:41.649620Z","shell.execute_reply.started":"2024-06-27T15:40:21.215386Z","shell.execute_reply":"2024-06-27T17:43:41.646844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"records_df_2 = pd.DataFrame(records_2)\nrecords_df_2['Precision'] = records_df.apply(lambda x: (x['TP']) / (x['TP']+x['FP']), axis=1)\nrecords_df_2['Recall'] = records_df.apply(lambda x: (x['TP']) / (x['TP']+x['FN']), axis=1)\nrecords_df_2","metadata":{"execution":{"iopub.status.busy":"2024-06-27T17:43:41.652834Z","iopub.execute_input":"2024-06-27T17:43:41.653292Z","iopub.status.idle":"2024-06-27T17:43:41.796604Z","shell.execute_reply.started":"2024-06-27T17:43:41.653256Z","shell.execute_reply":"2024-06-27T17:43:41.795575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics_list=['AUC', 'Cross_Entropy_Loss','Accuracy','Precision','Recall']\nprint_metrics_info(records_df, metrics_list)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T17:43:41.798087Z","iopub.execute_input":"2024-06-27T17:43:41.798512Z","iopub.status.idle":"2024-06-27T17:43:41.806181Z","shell.execute_reply.started":"2024-06-27T17:43:41.798473Z","shell.execute_reply":"2024-06-27T17:43:41.804815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion\n\nThe second trial for training did not improve. Due to the limitation of the deadline, we will not do another trial for training.\n\nTo improve the model's performance, the following techniques are worth trying: \n1. __Transform some numerical features into categorical ones:__ Take `hour_time` for instance, we directly use the round down hour and treat it as a numerical feature, which might be confusing during the training process. It is more reasonable to regard it as a categorical feature.\n2. __Try other approaches for feature re-selection__: Such as using __Permutation Feature Importance__.\n3. __Try other encoding approaches for categorical features.__","metadata":{}},{"cell_type":"markdown","source":"# Prepare to submit\n\nWe will adopt the model of the first trial for submission. ","metadata":{}},{"cell_type":"code","source":"X_test = test_df[selected_feats].to_pandas()\ny_test = model.predict_proba(X_test)[:,1]\nsubmission_df = pd.concat([test_df['id'].to_pandas(),\n                           pd.Series(y_test).rename('click')], axis=1)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T13:55:22.131274Z","iopub.execute_input":"2024-06-27T13:55:22.131772Z","iopub.status.idle":"2024-06-27T14:14:11.551119Z","shell.execute_reply.started":"2024-06-27T13:55:22.131728Z","shell.execute_reply":"2024-06-27T14:14:11.550057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('./submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T14:14:11.552586Z","iopub.execute_input":"2024-06-27T14:14:11.552953Z","iopub.status.idle":"2024-06-27T14:14:11.559757Z","shell.execute_reply.started":"2024-06-27T14:14:11.552924Z","shell.execute_reply":"2024-06-27T14:14:11.558777Z"},"trusted":true},"execution_count":null,"outputs":[]}]}