{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":108426,"databundleVersionId":13143440,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":134.274237,"end_time":"2025-07-28T14:48:33.155057","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-07-28T14:46:18.88082","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{"_cell_guid":"e845153f-3f32-44e3-9154-ff69481c0e4e","_uuid":"98fc5d72-a7ce-4d4b-aace-a2438071b245","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.008855,"end_time":"2025-07-28T14:46:23.80922","exception":false,"start_time":"2025-07-28T14:46:23.800365","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#install scikit learn 1.5.2 as this version supports root_mean_squared_log_error\n!pip uninstall scikit-learn -y\n!pip install -q scikit-learn==1.5.2","metadata":{"_cell_guid":"dbe5174f-84d0-46da-b508-574b0dfc7589","_uuid":"d881f7d9-04dd-4951-8cfc-4d88d3d4867d","execution":{"iopub.status.busy":"2025-07-30T15:30:37.910929Z","iopub.execute_input":"2025-07-30T15:30:37.911186Z","iopub.status.idle":"2025-07-30T15:30:48.673372Z","shell.execute_reply.started":"2025-07-30T15:30:37.911164Z","shell.execute_reply":"2025-07-30T15:30:48.672430Z"},"jupyter":{"source_hidden":true},"papermill":{"duration":10.891382,"end_time":"2025-07-28T14:46:34.708638","exception":false,"start_time":"2025-07-28T14:46:23.817256","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sklearn\nsklearn.__version__","metadata":{"_cell_guid":"a252d78a-a234-4055-a678-67b8a7708444","_uuid":"f38fd5c5-f6a9-4e3d-8274-95338b38a0cf","execution":{"iopub.status.busy":"2025-07-30T15:30:48.674582Z","iopub.execute_input":"2025-07-30T15:30:48.674898Z","iopub.status.idle":"2025-07-30T15:30:49.730045Z","shell.execute_reply.started":"2025-07-30T15:30:48.674868Z","shell.execute_reply":"2025-07-30T15:30:49.729225Z"},"jupyter":{"source_hidden":true},"papermill":{"duration":3.409736,"end_time":"2025-07-28T14:46:38.126991","exception":false,"start_time":"2025-07-28T14:46:34.717255","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport matplotlib.gridspec as gridspec\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"seaborn\")\nwarnings.filterwarnings(\"ignore\", category=FutureWarning, module=\"seaborn\")\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import root_mean_squared_log_error,mean_squared_error, mean_absolute_error, r2_score\n\nimport optuna\nimport lightgbm as lgb\n\nimport torch\nfrom sklearn.pipeline import Pipeline","metadata":{"_cell_guid":"a0b8af29-0ef7-4307-86b7-a6a49d68c2f8","_uuid":"fbe5a006-abe3-44f8-9180-e0b7876a3b68","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:30:49.731718Z","iopub.execute_input":"2025-07-30T15:30:49.732143Z","iopub.status.idle":"2025-07-30T15:31:16.457949Z","shell.execute_reply.started":"2025-07-30T15:30:49.732120Z","shell.execute_reply":"2025-07-30T15:31:16.456974Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":28.347096,"end_time":"2025-07-28T14:47:06.483415","exception":false,"start_time":"2025-07-28T14:46:38.136319","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read data","metadata":{"_cell_guid":"76e1b779-f2e0-42f2-9769-5bb2ea34340d","_uuid":"24ca4db4-18eb-4010-b587-0c6d908530d9","collapsed":false,"execution":{"iopub.execute_input":"2025-04-22T10:30:14.640596Z","iopub.status.busy":"2025-04-22T10:30:14.640324Z","iopub.status.idle":"2025-04-22T10:30:22.7682Z","shell.execute_reply":"2025-04-22T10:30:22.767482Z","shell.execute_reply.started":"2025-04-22T10:30:14.640576Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.016492,"end_time":"2025-07-28T14:47:06.512892","exception":false,"start_time":"2025-07-28T14:47:06.4964","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df=pd.read_csv('/kaggle/input/big-oai-final-course-1/train.csv')\ntest_df=pd.read_csv('/kaggle/input/big-oai-final-course-1/test.csv')","metadata":{"_cell_guid":"ec4014a3-ee2d-4abc-9707-575f5108dc42","_uuid":"0e555592-5079-440a-ab0d-fd8550523d18","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:16.461915Z","iopub.execute_input":"2025-07-30T15:31:16.462142Z","iopub.status.idle":"2025-07-30T15:31:25.270422Z","shell.execute_reply.started":"2025-07-30T15:31:16.462122Z","shell.execute_reply":"2025-07-30T15:31:25.269461Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":9.435514,"end_time":"2025-07-28T14:47:15.962261","exception":false,"start_time":"2025-07-28T14:47:06.526747","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploratory Data Analysis (EDA)","metadata":{"_cell_guid":"3b032f80-f02e-4797-a0f9-d6ae018d0a29","_uuid":"605ae8a4-71ba-4f81-a24e-a02bbb95ba40","collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T11:33:21.40058Z","iopub.status.idle":"2025-04-22T11:33:21.400991Z","shell.execute_reply":"2025-04-22T11:33:21.40082Z","shell.execute_reply.started":"2025-04-22T11:33:21.400803Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.008002,"end_time":"2025-07-28T14:47:15.978807","exception":false,"start_time":"2025-07-28T14:47:15.970805","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Xem qua vài thông số cơ bản của data","metadata":{"_cell_guid":"f32f5083-5350-41d0-b3d0-5e57cfb263f9","_uuid":"03400537-fb41-4fe5-bafa-0ffa533f38b4","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.008932,"end_time":"2025-07-28T14:47:15.995868","exception":false,"start_time":"2025-07-28T14:47:15.986936","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Check dataset shape and first rows\nprint(f\"Dataset contains {train_df.shape[0]} rows and {train_df.shape[1]} columns.\")\ntrain_df.head()","metadata":{"_cell_guid":"fd9b4339-9cbe-41be-b659-3c2281ae4ada","_uuid":"878a6bd7-5b05-45b6-aa5d-e64a97edc8d0","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:25.271375Z","iopub.execute_input":"2025-07-30T15:31:25.271715Z","iopub.status.idle":"2025-07-30T15:31:25.317221Z","shell.execute_reply.started":"2025-07-30T15:31:25.271688Z","shell.execute_reply":"2025-07-30T15:31:25.316269Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.059241,"end_time":"2025-07-28T14:47:16.064145","exception":false,"start_time":"2025-07-28T14:47:16.004904","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"_cell_guid":"cb4940db-190a-4189-903d-d976f2ea8a14","_uuid":"532187ec-c6c6-40c6-9e69-ffe173a9f086","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:25.318123Z","iopub.execute_input":"2025-07-30T15:31:25.318375Z","iopub.status.idle":"2025-07-30T15:31:26.062149Z","shell.execute_reply.started":"2025-07-30T15:31:25.318355Z","shell.execute_reply":"2025-07-30T15:31:26.061305Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.719254,"end_time":"2025-07-28T14:47:16.792282","exception":false,"start_time":"2025-07-28T14:47:16.073028","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### **Key Highlights**:\n- **Features**: 20 tính năng (không tính biến mục tiêu), bao gồm các loại dữ liệu số, phân loại và thời gian.\n- **Target Variable**: Số tiền bảo hiểm (Premium Amount), một giá trị số liên tục, được sử dụng làm mục tiêu để dự đoán.\n\n\n#### **Feature Breakdown**:\n1. **Numerical Features**:\n     - **Age**: Đại diện cho độ tuổi của người tham gia bảo hiểm.\n     - **Annual Income**: Chỉ ra khả năng tài chính của người tham gia.\n     - **Health Score**: Đo lường tình trạng sức khỏe của cá nhân.\n     - **Credit Score**: Đánh giá độ tin cậy tài chính.\n     - **Vehicle Age**: Đại diện cho độ tuổi của xe được bảo hiểm.\n2. **Categorical Features**:\n   - Qualitative characteristics, including:\n     - **Gender**, **Marital Status**, **Education Level**, **Occupation**, and **Policy Type**: Giới tính, Tình trạng hôn nhân, Trình độ học vấn, Nghề nghiệp, và Loại hợp đồng bảo hiểm, cung cấp thông tin về nhân khẩu học và ngữ cảnh.\n3. **Temporal Feature**:\n   - **Policy Start Date**: Thời điểm bắt đầu hợp đồng bảo hiểm.\n\n#### **Missing Values**:\n   - Một số tính năng có dữ liệu thiếu, cần các kỹ thuật thay thế giá trị thiếu phù hợp để duy trì tính toàn vẹn của dữ liệu.","metadata":{"_cell_guid":"f22a7457-c65b-4d30-bf2e-cb1c63c11019","_uuid":"ea03b8fc-6682-41b9-9d7a-239acdd483a9","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.008368,"end_time":"2025-07-28T14:47:16.809464","exception":false,"start_time":"2025-07-28T14:47:16.801096","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df.describe().round(2)","metadata":{"_cell_guid":"2c1ff42b-dd5f-440f-bc59-161f8e8eb2b2","_uuid":"f3213d70-a406-42bc-9e55-8038558251d1","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:26.063053Z","iopub.execute_input":"2025-07-30T15:31:26.063318Z","iopub.status.idle":"2025-07-30T15:31:26.711767Z","shell.execute_reply.started":"2025-07-30T15:31:26.063290Z","shell.execute_reply":"2025-07-30T15:31:26.710741Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.687069,"end_time":"2025-07-28T14:47:17.505092","exception":false,"start_time":"2025-07-28T14:47:16.818023","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save 'id' column for submission\ntest_ids = test_df['id']\n\n# Define the target column\ntarget_column = 'Premium Amount'\n\n# Select categorical and numerical columns (initial)\ncategorical_columns = train_df.select_dtypes(include=['object']).columns\nnumerical_columns = train_df.select_dtypes(exclude=['object']).columns\n\n# Print out column information\nprint(\"Target Column:\", target_column)\nprint(\"\\nCategorical Columns:\", categorical_columns.tolist())\nprint(\"\\nNumerical Columns:\", numerical_columns.tolist())","metadata":{"_cell_guid":"15fea608-dd97-4f4c-818f-ed7ece6e6378","_uuid":"8a9ba8ba-b943-4eef-91ef-48ef3b0a391c","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:26.716659Z","iopub.execute_input":"2025-07-30T15:31:26.717071Z","iopub.status.idle":"2025-07-30T15:31:26.921273Z","shell.execute_reply.started":"2025-07-30T15:31:26.717039Z","shell.execute_reply":"2025-07-30T15:31:26.920135Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.216325,"end_time":"2025-07-28T14:47:17.73181","exception":false,"start_time":"2025-07-28T14:47:17.515485","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for column in categorical_columns:\n    num_unique = train_df[column].nunique()\n    print(f\"'{column}' has {num_unique} unique categories.\")","metadata":{"_cell_guid":"7aae254d-1442-4290-b22c-8454e6b36608","_uuid":"0f94e2c7-b0dd-4f75-b9aa-7d2d99002f96","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:26.922118Z","iopub.execute_input":"2025-07-30T15:31:26.922356Z","iopub.status.idle":"2025-07-30T15:31:27.828138Z","shell.execute_reply.started":"2025-07-30T15:31:26.922336Z","shell.execute_reply":"2025-07-30T15:31:27.827238Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.962273,"end_time":"2025-07-28T14:47:18.705509","exception":false,"start_time":"2025-07-28T14:47:17.743236","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Cột Policy Start Date là 1 cột đặc biệt mà ta cần giải quyết sau","metadata":{"_cell_guid":"dcbcca84-98b2-4c13-b617-f0ad1b245bc2","_uuid":"c3253db0-ab9a-408e-b72a-33b7454fc173","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.009467,"end_time":"2025-07-28T14:47:18.724434","exception":false,"start_time":"2025-07-28T14:47:18.714967","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Visualize","metadata":{"_cell_guid":"e989d6fc-9bd1-4caa-b72f-03694b00b9a7","_uuid":"1b73e1e3-1ca0-49a3-90f4-e5714f1c046d","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.076872,"end_time":"2025-07-28T14:47:18.811053","exception":false,"start_time":"2025-07-28T14:47:18.734181","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(15,9))\nplt.title(\"Visualizing Missing Values\")\nsns.heatmap(train_df.isnull(), cbar=False, cmap=sns.color_palette('magma'), yticklabels=False);\nplt.show()","metadata":{"_cell_guid":"b0d25802-cd55-4719-9d8d-ee5973be5d64","_uuid":"9b879092-39c4-4fe7-bd3d-b4dd70e387d5","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:27.829151Z","iopub.execute_input":"2025-07-30T15:31:27.829466Z","iopub.status.idle":"2025-07-30T15:31:52.059112Z","shell.execute_reply.started":"2025-07-30T15:31:27.829439Z","shell.execute_reply":"2025-07-30T15:31:52.058002Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":24.512002,"end_time":"2025-07-28T14:47:43.332476","exception":false,"start_time":"2025-07-28T14:47:18.820474","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to calculate missing values, percentages, and data types\ndef missing_values_table(df):\n    missing_count = df.isnull().sum()\n    missing_percentage = 100 * missing_count / len(df)\n    data_types = df.dtypes\n    return pd.DataFrame({\n        'Missing Values': missing_count,\n        'Percentage (%)': missing_percentage,\n        'Data Type': data_types\n    })\n\n# Create tables for train and test datasets\ntrain_missing_table = missing_values_table(train_df)\ntest_missing_table = missing_values_table(test_df)\n\n# Display the tables\nprint(\"Missing Values Table - Training Dataset:\\n\")\ndisplay(train_missing_table[train_missing_table['Missing Values'] > 0])  # Display only features with missing values\nprint(\"\\n\")\n\nprint(\"Missing Values Table - Test Dataset:\\n\")\ndisplay(test_missing_table[test_missing_table['Missing Values'] > 0])","metadata":{"_cell_guid":"b946de49-d23f-45cb-962d-386662cc9f7c","_uuid":"0044f9aa-c28f-45cc-87bb-d5a8846ffda3","execution":{"iopub.status.busy":"2025-07-30T15:31:52.060043Z","iopub.execute_input":"2025-07-30T15:31:52.060373Z","iopub.status.idle":"2025-07-30T15:31:52.922292Z","shell.execute_reply.started":"2025-07-30T15:31:52.060344Z","shell.execute_reply":"2025-07-30T15:31:52.921587Z"},"jupyter":{"source_hidden":true},"papermill":{"duration":0.882136,"end_time":"2025-07-28T14:47:44.225258","exception":false,"start_time":"2025-07-28T14:47:43.343122","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Previous Claims\tvà Occupation có nhiều missing values nhất!","metadata":{"_cell_guid":"9d01f504-baab-403b-8969-c6d7e7095384","_uuid":"5eef9808-a040-4741-9207-4cf4720be943","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.01028,"end_time":"2025-07-28T14:47:44.246292","exception":false,"start_time":"2025-07-28T14:47:44.236012","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Create a color palette for the columns\npalette = sns.color_palette('tab10', len(numerical_columns))\ncolor_dict = dict(zip(numerical_columns, palette))\n\n# Create a grid of subplots for histograms and boxplots only\nfig = plt.figure(figsize=(15, 10 * len(numerical_columns)))  # Adjusted width since only one column of plots\ngs = gridspec.GridSpec(2 * len(numerical_columns), 1, figure=fig)  # Single column grid\n\nfor i, column in enumerate(numerical_columns):\n    if(column==\"id\"): continue\n    if train_df[column].nunique() > 50:\n        discrete = False\n    else:\n        discrete = True\n    \n    # Plot histogram with a unique color\n    ax_hist = fig.add_subplot(gs[2 * i, 0])\n    sns.histplot(\n        data=train_df, x=column, fill=True, common_norm=False, alpha=0.6,\n        linewidth=0.8, color=color_dict[column], ax=ax_hist, discrete=discrete\n    )\n    \n    # Plot boxplot with the same unique color\n    ax_box = fig.add_subplot(gs[2 * i + 1, 0])\n    sns.boxplot(data=train_df, x=column, ax=ax_box, color=color_dict[column])\n    ax_box.set_title(f'{column} (Boxplot)', fontsize=14)\n    sns.despine(ax=ax_box)\n\nplt.tight_layout()  # Adjust subplots to fit into the figure area\nplt.show()","metadata":{"_cell_guid":"99402180-cad7-446d-9d76-7c10bdaa762b","_uuid":"5e430da0-bb1f-459d-a11f-78e6798897a0","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:31:52.923144Z","iopub.execute_input":"2025-07-30T15:31:52.923428Z","iopub.status.idle":"2025-07-30T15:32:01.350856Z","shell.execute_reply.started":"2025-07-30T15:31:52.923407Z","shell.execute_reply":"2025-07-30T15:32:01.349817Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":8.745834,"end_time":"2025-07-28T14:47:53.003147","exception":false,"start_time":"2025-07-28T14:47:44.257313","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Một vài cột bị lệch phải:\n+ Annual Income\n+ Premium Amount","metadata":{"_cell_guid":"fab33932-14df-4959-a88f-6190c9577a66","_uuid":"87b1ac73-d863-4eb9-b210-65c41b1a305c","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.017061,"end_time":"2025-07-28T14:47:53.037778","exception":false,"start_time":"2025-07-28T14:47:53.020717","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Filtrer les colonnes catégorielles et exclure 'Policy Start Date'\nfiltered_columns = [col for col in categorical_columns if col != 'Policy Start Date']\n\n# Créer des sous-graphiques pour barplots et boxplots\nfig, axes = plt.subplots(len(filtered_columns), 2, figsize=(15, 5 * len(filtered_columns)))\n\nfor i, column in enumerate(filtered_columns):\n    # Barplot à gauche\n    sns.countplot(data=train_df, x=column, ax=axes[i, 0], palette='tab10')\n    axes[i, 0].set_title(f'Distribution of {column}', fontsize=14)\n    axes[i, 0].set_xlabel(column, fontsize=12)\n    axes[i, 0].set_ylabel('Count', fontsize=12)\n    sns.despine(ax=axes[i, 0])\n\n    # Boxplot à droite\n    sns.boxplot(data=train_df, x=column, y=target_column, ax=axes[i, 1], palette='tab10')\n    axes[i, 1].set_title(f'{column} vs {target_column}', fontsize=14)\n    axes[i, 1].set_xlabel(column, fontsize=12)\n    axes[i, 1].set_ylabel(target_column, fontsize=12)\n    sns.despine(ax=axes[i, 1])\n\n\nplt.tight_layout()  # Ajustement global des sous-graphiques\nplt.show()","metadata":{"_cell_guid":"d3301f31-c3e8-42fe-88e0-3add9ba7ec26","_uuid":"c003f2ab-b94a-497b-87f7-0004618812d1","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:01.351909Z","iopub.execute_input":"2025-07-30T15:32:01.352590Z","iopub.status.idle":"2025-07-30T15:32:17.125974Z","shell.execute_reply.started":"2025-07-30T15:32:01.352564Z","shell.execute_reply":"2025-07-30T15:32:17.124606Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":15.988128,"end_time":"2025-07-28T14:48:09.042295","exception":false,"start_time":"2025-07-28T14:47:53.054167","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Các categorial features phân bố khá đều","metadata":{"_cell_guid":"d91caaff-9cd5-48be-9f2a-417f9840d9de","_uuid":"6e7c2c75-0c97-4b13-a3a0-561a62c3273f","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.025377,"end_time":"2025-07-28T14:48:09.093398","exception":false,"start_time":"2025-07-28T14:48:09.068021","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{"_cell_guid":"3692717c-7bcb-4412-88c7-39a3f886427a","_uuid":"cac61d50-6f61-41d0-8329-2eafaa2b0713","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.024496,"end_time":"2025-07-28T14:48:09.142292","exception":false,"start_time":"2025-07-28T14:48:09.117796","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Date Handling","metadata":{"_cell_guid":"025364e0-6019-4ab6-bff5-cbaec4f7df5e","_uuid":"25c1b2bb-4126-43e5-99b3-4cc7f687b0c7","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.024542,"end_time":"2025-07-28T14:48:09.191592","exception":false,"start_time":"2025-07-28T14:48:09.16705","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def date(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Month_name'] = df['Policy Start Date'].dt.month_name()\n    df['Day_of_week'] = df['Policy Start Date'].dt.day_name()\n    df['Week'] = df['Policy Start Date'].dt.isocalendar().week\n    df['Year_sin'] = np.sin(2 * np.pi * df['Year'])\n    df['Year_cos'] = np.cos(2 * np.pi * df['Year'])\n    min_year = df['Year'].min()\n    max_year = df['Year'].max()\n    df['Year_sin'] = np.sin(2 * np.pi * (df['Year'] - min_year) / (max_year - min_year))\n    df['Year_cos'] = np.cos(2 * np.pi * (df['Year'] - min_year) / (max_year - min_year))\n    df['Month_sin'] = np.sin(2 * np.pi * df['Month'] / 12) \n    df['Month_cos'] = np.cos(2 * np.pi * df['Month'] / 12)\n    df['Day_sin'] = np.sin(2 * np.pi * df['Day'] / 31)  \n    df['Day_cos'] = np.cos(2 * np.pi * df['Day'] / 31)\n    \n    df.drop('Policy Start Date', axis=1, inplace=True)\n\n    return df\n\n# Apply the date function to both datasets\ntrain_df = date(train_df)\ntest_df = date(test_df)","metadata":{"_cell_guid":"7b154879-a1d7-4903-bdf4-ebdd377e7cab","_uuid":"241e682c-4f99-4de8-8cb3-e88fa73f22a8","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:17.127499Z","iopub.execute_input":"2025-07-30T15:32:17.127839Z","iopub.status.idle":"2025-07-30T15:32:20.274488Z","shell.execute_reply.started":"2025-07-30T15:32:17.127801Z","shell.execute_reply":"2025-07-30T15:32:20.273459Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":2.136706,"end_time":"2025-07-28T14:48:11.352871","exception":false,"start_time":"2025-07-28T14:48:09.216165","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"+ `df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])`: Chuyển đổi cột 'Policy Start Date' sang định dạng datetime chuẩn của Pandas.\n\n+ `df['Year'], df['Day'], df['Month']`: tạo các cột ngày tháng năm\n+ `df['Month_name'] = df['Policy Start Date'].dt.month_name()`: Cung cấp một biểu diễn dạng category (phân loại) của tháng.\n+ `df['Day_of_week'] = df['Policy Start Date'].dt.day_name()`: Nắm bắt các quy luật theo tuần. Ví dụ: có thể có sự khác biệt về số lượng hợp đồng bắt đầu vào ngày thường so với cuối tuần.\n+ `df['Week']`: Tạo cột 'Week' chứa số thứ tự của tuần trong năm (theo chuẩn ISO, thường từ 1 đến 52 hoặc 53).\n+ Mã hóa Cyclic (Tuần hoàn) cho Năm, Tháng, Ngày: Ví dụ- Mã hóa sin/cos đặt các tháng lên một vòng tròn, thể hiện đúng tính chất \"gần nhau\" của tháng 12 và tháng 1, giúp mô hình hiểu được tính mùa vụ.","metadata":{"_cell_guid":"9d878954-4e5b-405f-90e7-3d8b3f9fb4fe","_uuid":"dcf0442b-2c36-4b55-9332-656cecfc2bde","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.024392,"end_time":"2025-07-28T14:48:11.401347","exception":false,"start_time":"2025-07-28T14:48:11.376955","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Define features and target\nnumerical_features = [\n    'Age', 'Annual Income', 'Number of Dependents', 'Health Score', \n    'Previous Claims', 'Vehicle Age', 'Credit Score', 'Insurance Duration', \n    'Year_sin', 'Year_cos', 'Month_sin', 'Month_cos', 'Day_sin', 'Day_cos'\n]\ncategorical_features = [\n    'Gender', 'Marital Status', 'Education Level', 'Occupation', 'Location',\n    'Policy Type', 'Customer Feedback', 'Smoking Status', 'Exercise Frequency', \n    'Property Type', 'Month_name', 'Day_of_week'\n]\ntarget_column = 'Premium Amount'","metadata":{"_cell_guid":"105ba385-56b3-431f-84f0-13092d2a2770","_uuid":"3672cfa4-3495-4245-bbf3-4f6841db9c02","execution":{"iopub.status.busy":"2025-07-30T15:32:20.275573Z","iopub.execute_input":"2025-07-30T15:32:20.275923Z","iopub.status.idle":"2025-07-30T15:32:20.281674Z","shell.execute_reply.started":"2025-07-30T15:32:20.275896Z","shell.execute_reply":"2025-07-30T15:32:20.280581Z"},"papermill":{"duration":0.03318,"end_time":"2025-07-28T14:48:11.459116","exception":false,"start_time":"2025-07-28T14:48:11.425936","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## NEW: Skewness Reduction with Log Transformation","metadata":{}},{"cell_type":"code","source":"# Log-transform skewed features\ntrain_df['Annual Income'] = np.log1p(train_df['Annual Income'])\ntest_df['Annual Income'] = np.log1p(test_df['Annual Income'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-30T15:32:20.282631Z","iopub.execute_input":"2025-07-30T15:32:20.282934Z","iopub.status.idle":"2025-07-30T15:32:20.322933Z","shell.execute_reply.started":"2025-07-30T15:32:20.282909Z","shell.execute_reply":"2025-07-30T15:32:20.321673Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Features and Target","metadata":{"_cell_guid":"10d0e716-1fe3-4bed-b523-161bfd9f7dbf","_uuid":"ef36bcdf-100d-43c5-b863-a0f4a7b2a764","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.023705,"end_time":"2025-07-28T14:48:11.507007","exception":false,"start_time":"2025-07-28T14:48:11.483302","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Split train data into features and target\nX = train_df.drop(columns=[target_column, 'id', 'Year', 'Month', 'Day', 'Week'])\ny = train_df[target_column]","metadata":{"_cell_guid":"580f3f4c-cb03-4941-bb83-e446beca1df8","_uuid":"7daa3012-8b74-402e-96b2-1662158ce8a0","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:20.324134Z","iopub.execute_input":"2025-07-30T15:32:20.324459Z","iopub.status.idle":"2025-07-30T15:32:20.803355Z","shell.execute_reply.started":"2025-07-30T15:32:20.324431Z","shell.execute_reply":"2025-07-30T15:32:20.802377Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.228856,"end_time":"2025-07-28T14:48:11.759724","exception":false,"start_time":"2025-07-28T14:48:11.530868","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Handle Missing Values & Preprocessing Pipeline","metadata":{"_cell_guid":"58e4d73c-4783-4cf7-bc13-b24f271abc8f","_uuid":"62ca5126-1558-49b7-a34b-62f2c33e3275","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.027272,"end_time":"2025-07-28T14:48:11.812523","exception":false,"start_time":"2025-07-28T14:48:11.785251","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Preprocessing pipeline for numerical features\nnum_pipeline = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='median')),\n])\n\n# Preprocessing pipeline for categorical features\ncat_pipeline = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='constant', fill_value='Unknown')),  # Handle missing values\n    ('onehot', OneHotEncoder(handle_unknown='ignore'))                      # Encode categorical features\n])\n\n# Combine pipelines into a ColumnTransformer\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', num_pipeline, numerical_features),\n        ('cat', cat_pipeline, categorical_features)\n    ]\n)\n\n# Preprocess train and test data\nX_processed = preprocessor.fit_transform(X)\ntest_processed = preprocessor.transform(test_df.drop(columns=['id', 'Year', 'Month', 'Day', 'Week']))","metadata":{"_cell_guid":"b0215473-c44e-4d53-9645-2fd5a3bafc9c","_uuid":"8f76dc0a-fa0b-4c5a-832d-6f555e1fc0d9","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:20.804460Z","iopub.execute_input":"2025-07-30T15:32:20.804809Z","iopub.status.idle":"2025-07-30T15:32:32.994959Z","shell.execute_reply.started":"2025-07-30T15:32:20.804780Z","shell.execute_reply":"2025-07-30T15:32:32.993961Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":11.492055,"end_time":"2025-07-28T14:48:23.330348","exception":false,"start_time":"2025-07-28T14:48:11.838293","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Splitting Data","metadata":{"_cell_guid":"2d10ece2-ce2c-455d-be80-6625ef9e35c4","_uuid":"2282b291-2f9c-44a0-b691-9e1cc8b0f40a","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.023774,"end_time":"2025-07-28T14:48:23.37815","exception":false,"start_time":"2025-07-28T14:48:23.354376","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Split the data\nX_train, X_val, y_train, y_val = train_test_split(X_processed, y, test_size=0.2, random_state=42)","metadata":{"_cell_guid":"cdb46707-06ff-47bb-9376-9c7f061443f0","_uuid":"5060e5ae-61e4-4193-b40f-e26c8846af75","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:32.996138Z","iopub.execute_input":"2025-07-30T15:32:32.996424Z","iopub.status.idle":"2025-07-30T15:32:33.416049Z","shell.execute_reply.started":"2025-07-30T15:32:32.996402Z","shell.execute_reply":"2025-07-30T15:32:33.415152Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.394056,"end_time":"2025-07-28T14:48:23.795974","exception":false,"start_time":"2025-07-28T14:48:23.401918","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training","metadata":{"_cell_guid":"388d81db-33be-4d4f-a2d3-548cf35b3866","_uuid":"038cc7b3-af03-4826-b7b5-40d64d2ff7d4","collapsed":false,"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.023544,"end_time":"2025-07-28T14:48:23.843822","exception":false,"start_time":"2025-07-28T14:48:23.820278","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\n\n# --- Model Imports ---\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.ensemble import RandomForestRegressor\nimport xgboost as xgb\nimport lightgbm as lgb","metadata":{"_cell_guid":"eedc67ea-59e7-403d-a516-27214131aae1","_uuid":"72d10576-d21a-4aee-a5fb-64451c6b92d2","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:33.417459Z","iopub.execute_input":"2025-07-30T15:32:33.417775Z","iopub.status.idle":"2025-07-30T15:32:33.721713Z","shell.execute_reply.started":"2025-07-30T15:32:33.417745Z","shell.execute_reply":"2025-07-30T15:32:33.720840Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.31468,"end_time":"2025-07-28T14:48:24.182158","exception":false,"start_time":"2025-07-28T14:48:23.867478","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. Linear Regression ---\nprint(\"--- Training Linear Regression ---\")\n# Instantiate the model\nlr_model = LinearRegression()\n\n# Train the model\nlr_model.fit(X_train, y_train)\nprint(\"Linear Regression training complete.\")\n\n# Predict on the validation set\ny_pred_lr = lr_model.predict(X_val)\nprint(\"-\" * 30)\n\n\n# --- 2. Ridge Regression ---\nprint(\"--- Training Ridge Regression ---\")\n# Instantiate the model (alpha is the regularization strength)\n# A common default/starting point is alpha=1.0. You might tune this later.\nridge_model = Ridge(alpha=1.0)\n\n# Train the model\nridge_model.fit(X_train, y_train)\nprint(\"Ridge Regression training complete.\")\n\n# Predict on the validation set\ny_pred_ridge = ridge_model.predict(X_val)\nprint(\"-\" * 30)\n\n\n# --- 3. Lasso Regression ---\nprint(\"--- Training Lasso Regression ---\")\n# Instantiate the model (alpha is the regularization strength)\n# A common default/starting point is alpha=1.0. You might tune this later.\nlasso_model = Lasso(alpha=1.0)\n\n# Train the model\nlasso_model.fit(X_train, y_train)\nprint(\"Lasso Regression training complete.\")\n\n# Predict on the validation set\ny_pred_lasso = lasso_model.predict(X_val)\nprint(\"-\" * 30)\n\n# --- 4. XGBoost Regressor ---\nprint(\"--- Training XGBoost Regressor ---\")\nxgb_model = xgb.XGBRegressor(objective='reg:squarederror', \n                             random_state=42)\nxgb_model.fit(X_train, y_train)\nprint(\"Training XGBoost complete.\")\ny_pred_xgb = xgb_model.predict(X_val)\nprint(\"-\" * 30)\n\n# --- 5. LightGBM Regressor ---\nprint(\"--- Training LightGBM Regressor ---\")\nlgb_model = lgb.LGBMRegressor(random_state=42)\nlgb_model.fit(X_train, y_train)\nprint(\"Training LightGBM complete.\")\ny_pred_lgb = lgb_model.predict(X_val)\nprint(\"-\" * 30)","metadata":{"_cell_guid":"39d3dc76-6db4-45a2-a7e4-0130323b7b08","_uuid":"4134e630-a1cf-4c7d-9d66-ab35be177184","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:32:33.722654Z","iopub.execute_input":"2025-07-30T15:32:33.724065Z","iopub.status.idle":"2025-07-30T15:33:04.060118Z","shell.execute_reply.started":"2025-07-30T15:32:33.724037Z","shell.execute_reply":"2025-07-30T15:33:04.058792Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":4.255768,"end_time":"2025-07-28T14:48:28.462244","exception":false,"start_time":"2025-07-28T14:48:24.206476","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import mean_squared_log_error\n\n# Assumes y_val, y_pred_lr, y_pred_ridge, y_pred_lasso are already defined.\n\ndef calculate_rmsle(y_true, y_pred):\n    \"\"\"Calculates RMSLE, clipping negative predictions to 0.\"\"\"\n    # Ensure no negative true values (RMSLE undefined)\n    if np.any(y_true < 0):\n        print(f\"Warning: Negative true values found. RMSLE calculation may fail or be invalid.\")\n        # Optionally return NaN or raise error, here we proceed but warn.\n\n    # Clip negative predictions to 0\n    y_pred_clipped = np.maximum(y_pred, 0)\n\n    msle = mean_squared_log_error(y_true, y_pred_clipped)\n    rmsle = np.sqrt(msle)\n    return rmsle\n\nprint(\"--- Evaluating Models using RMSLE ---\")\n\n# Evaluate Linear Regression\nrmsle_lr = calculate_rmsle(y_val, y_pred_lr)\nprint(f\"Linear Regression RMSLE: {rmsle_lr:.4f}\")\n\n# Evaluate Ridge Regression\nrmsle_ridge = calculate_rmsle(y_val, y_pred_ridge)\nprint(f\"Ridge Regression RMSLE:  {rmsle_ridge:.4f}\")\n\n# Evaluate Lasso Regression\nrmsle_lasso = calculate_rmsle(y_val, y_pred_lasso)\nprint(f\"Lasso Regression RMSLE:  {rmsle_lasso:.4f}\")\n\nrmsle_xgb = calculate_rmsle(y_val, y_pred_xgb)\nprint(f\"XGBoost Regressor RMSLE: {rmsle_xgb:.4f}\")\n\nrmsle_lgb = calculate_rmsle(y_val, y_pred_lgb)\nprint(f\"LightGBM Regressor RMSLE:{rmsle_lgb:.4f}\")\n\nprint(\"-\" * 30)","metadata":{"_cell_guid":"1d28e886-c25e-4158-b648-6fc0c373afec","_uuid":"09e1436e-1bc4-4ddf-aab2-ea99ebc38150","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:33:04.061332Z","iopub.execute_input":"2025-07-30T15:33:04.062280Z","iopub.status.idle":"2025-07-30T15:33:04.105569Z","shell.execute_reply.started":"2025-07-30T15:33:04.062246Z","shell.execute_reply":"2025-07-30T15:33:04.104741Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.057421,"end_time":"2025-07-28T14:48:28.568002","exception":false,"start_time":"2025-07-28T14:48:28.510581","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predict","metadata":{"papermill":{"duration":0.023667,"end_time":"2025-07-28T14:48:28.618066","exception":false,"start_time":"2025-07-28T14:48:28.594399","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"/kaggle/input/big-oai-final-course-1/sample_submission.csv\")\nsample_submission.head()","metadata":{"_cell_guid":"b7f9a259-5359-4956-a09a-d547e2f9c972","_uuid":"487d52a0-904e-4278-accd-adbd51764d70","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:33:04.106963Z","iopub.execute_input":"2025-07-30T15:33:04.107304Z","iopub.status.idle":"2025-07-30T15:33:04.228573Z","shell.execute_reply.started":"2025-07-30T15:33:04.107272Z","shell.execute_reply":"2025-07-30T15:33:04.227778Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.133337,"end_time":"2025-07-28T14:48:28.7754","exception":false,"start_time":"2025-07-28T14:48:28.642063","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict\ny_pred = xgb_model.predict(test_processed)","metadata":{"_cell_guid":"86580603-044d-4898-82b3-8577e31179cc","_uuid":"7be0a261-1300-42ef-9040-e75d9c788798","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:33:04.229464Z","iopub.execute_input":"2025-07-30T15:33:04.229696Z","iopub.status.idle":"2025-07-30T15:33:04.641271Z","shell.execute_reply.started":"2025-07-30T15:33:04.229677Z","shell.execute_reply":"2025-07-30T15:33:04.640448Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.058103,"end_time":"2025-07-28T14:48:28.858132","exception":false,"start_time":"2025-07-28T14:48:28.800029","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.DataFrame()\n\nif 'id' in test_df.columns:\n    submission_df['id'] = test_df['id'].values\nelse:\n    submission_df['id'] = sample_submission['id'].values\n\ntarget_column_name = sample_submission.columns[1] # Assumes target is the second column\nsubmission_df[target_column_name] = y_pred\n\n# Save\nsubmission_filename = 'submission.csv'\nsubmission_df.to_csv(submission_filename, index=False)\n\nprint(f\"Submission file '{submission_filename}' created.\")\nprint(submission_df.head())","metadata":{"_cell_guid":"11a7f903-b686-4647-8927-497223b5e336","_uuid":"0fc7924a-bf87-4690-9743-5d643165ba55","collapsed":false,"execution":{"iopub.status.busy":"2025-07-30T15:33:04.642412Z","iopub.execute_input":"2025-07-30T15:33:04.642754Z","iopub.status.idle":"2025-07-30T15:33:05.135167Z","shell.execute_reply.started":"2025-07-30T15:33:04.642707Z","shell.execute_reply":"2025-07-30T15:33:05.134395Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.722169,"end_time":"2025-07-28T14:48:29.608349","exception":false,"start_time":"2025-07-28T14:48:28.88618","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}