{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\nfile_path = '/kaggle/input/brist1d/test.csv'\ntest_data = pd.read_csv(file_path)\n\nnull_values_summary = test_data.isnull().sum()\nprint(null_values_summary[null_values_summary > 0])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:07.055675Z","iopub.execute_input":"2024-11-19T16:04:07.057224Z","iopub.status.idle":"2024-11-19T16:04:07.347254Z","shell.execute_reply.started":"2024-11-19T16:04:07.057177Z","shell.execute_reply":"2024-11-19T16:04:07.345966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_rows = test_data[test_data.isnull().any(axis=1)]\n\nprint(null_rows)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:07.349809Z","iopub.execute_input":"2024-11-19T16:04:07.350173Z","iopub.status.idle":"2024-11-19T16:04:07.449181Z","shell.execute_reply.started":"2024-11-19T16:04:07.350140Z","shell.execute_reply":"2024-11-19T16:04:07.447741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg_column_null = test_data[test_data['bg-5:55'].isnull()]\nprint(bg_column_null)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:07.451091Z","iopub.execute_input":"2024-11-19T16:04:07.451596Z","iopub.status.idle":"2024-11-19T16:04:07.482318Z","shell.execute_reply.started":"2024-11-19T16:04:07.451546Z","shell.execute_reply":"2024-11-19T16:04:07.480827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_null_values = test_data.isnull().sum().sum()\nprint(f\"Total null values: {total_null_values}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:07.484172Z","iopub.execute_input":"2024-11-19T16:04:07.484580Z","iopub.status.idle":"2024-11-19T16:04:07.508350Z","shell.execute_reply.started":"2024-11-19T16:04:07.484543Z","shell.execute_reply":"2024-11-19T16:04:07.506864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\ntest_data = pd.read_csv('/kaggle/input/brist1d/test.csv')\nnull_counts = test_data.isnull().sum()\nnull_counts = null_counts[null_counts > 0]\n\nplt.figure(figsize=(16, 8))  \nnull_counts.plot(kind='bar', color='skyblue')\nplt.title('Null Values per Column in test.csv')\nplt.xlabel('Columns')\nplt.ylabel('Number of Null Values')\n\n\nplt.xticks(rotation=90, ha='center', fontsize=8)\nplt.gca().set_xticks(plt.gca().get_xticks()[::5])  \n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:07.512940Z","iopub.execute_input":"2024-11-19T16:04:07.513492Z","iopub.status.idle":"2024-11-19T16:04:10.315144Z","shell.execute_reply.started":"2024-11-19T16:04:07.513439Z","shell.execute_reply":"2024-11-19T16:04:10.313661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n\ntest_data = pd.read_csv('/kaggle/input/brist1d/test.csv')\n\nthreshold = 0.2 * test_data.shape[0]\n\nfiltered_data = test_data.loc[:, test_data.isnull().sum() < threshold]\n\n\nprint(f\"Original shape: {test_data.shape}\")\nprint(f\"Filtered shape: {filtered_data.shape}\")\n\nfiltered_data.to_csv('/kaggle/working/filtered_test.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:10.316781Z","iopub.execute_input":"2024-11-19T16:04:10.317122Z","iopub.status.idle":"2024-11-19T16:04:11.190201Z","shell.execute_reply.started":"2024-11-19T16:04:10.317090Z","shell.execute_reply":"2024-11-19T16:04:11.188624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ntest_data = pd.read_csv('/kaggle/input/brist1d/test.csv')\n\nthreshold = 0.2 * test_data.shape[0]\n\n\nfiltered_data = test_data.loc[:, test_data.isnull().sum() < threshold]\nnull_counts_filtered = filtered_data.isnull().sum()\n\n\nplt.figure(figsize=(15, 8))\nnull_counts_filtered.plot(kind='bar', color='green')\nplt.title('Null Values per Column (Filtered Data)', fontsize=16)\nplt.xlabel('Columns', fontsize=12)\nplt.ylabel('Number of Null Values', fontsize=12)\n\nplt.xticks(\n    range(len(null_counts_filtered)),  \n    [label if i % 10 == 0 else '' for i, label in enumerate(null_counts_filtered.index)],  # Show every 10th label\n    rotation=90,\n    ha='center',\n    fontsize=10\n)\n\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:11.191930Z","iopub.execute_input":"2024-11-19T16:04:11.192301Z","iopub.status.idle":"2024-11-19T16:04:12.577310Z","shell.execute_reply.started":"2024-11-19T16:04:11.192266Z","shell.execute_reply":"2024-11-19T16:04:12.575812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n\ntrain_data = pd.read_csv('/kaggle/input/brist1d/train.csv')\n\n\nnull_counts_train = train_data.isnull().sum()\n\nnull_counts_train = null_counts_train[null_counts_train > 0]\n\nprint(\"Columns with Null Values in train.csv:\")\nprint(null_counts_train)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:12.579419Z","iopub.execute_input":"2024-11-19T16:04:12.580504Z","iopub.status.idle":"2024-11-19T16:04:23.724041Z","shell.execute_reply.started":"2024-11-19T16:04:12.580451Z","shell.execute_reply":"2024-11-19T16:04:23.722674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_rows = train_data[train_data.isnull().any(axis=1)]\n\nprint(null_rows)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:23.726040Z","iopub.execute_input":"2024-11-19T16:04:23.726537Z","iopub.status.idle":"2024-11-19T16:04:24.774050Z","shell.execute_reply.started":"2024-11-19T16:04:23.726486Z","shell.execute_reply":"2024-11-19T16:04:24.772515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg_column_null = train_data[train_data['bg-5:55'].isnull()]\nprint(bg_column_null)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:24.775755Z","iopub.execute_input":"2024-11-19T16:04:24.776122Z","iopub.status.idle":"2024-11-19T16:04:24.859157Z","shell.execute_reply.started":"2024-11-19T16:04:24.776087Z","shell.execute_reply":"2024-11-19T16:04:24.857735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_null_values = train_data.isnull().sum().sum()\nprint(f\"Total null values: {total_null_values}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:24.861439Z","iopub.execute_input":"2024-11-19T16:04:24.861953Z","iopub.status.idle":"2024-11-19T16:04:25.400765Z","shell.execute_reply.started":"2024-11-19T16:04:24.861903Z","shell.execute_reply":"2024-11-19T16:04:25.399547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nthreshold = 0.2 * len(train_data)\n\n\nfiltered_train_data = train_data.loc[:, train_data.isnull().sum() < threshold]\n\nprint(f\"Columns retained: {filtered_train_data.columns}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:25.402147Z","iopub.execute_input":"2024-11-19T16:04:25.402479Z","iopub.status.idle":"2024-11-19T16:04:26.068742Z","shell.execute_reply.started":"2024-11-19T16:04:25.402448Z","shell.execute_reply":"2024-11-19T16:04:26.067579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(filtered_train_data)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:26.070297Z","iopub.execute_input":"2024-11-19T16:04:26.070699Z","iopub.status.idle":"2024-11-19T16:04:26.120379Z","shell.execute_reply.started":"2024-11-19T16:04:26.070651Z","shell.execute_reply":"2024-11-19T16:04:26.118747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(filtered_train_data.info())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:26.127219Z","iopub.execute_input":"2024-11-19T16:04:26.127804Z","iopub.status.idle":"2024-11-19T16:04:26.147442Z","shell.execute_reply.started":"2024-11-19T16:04:26.127745Z","shell.execute_reply":"2024-11-19T16:04:26.145733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nthreshold = 0.2 * train_data.shape[0]\n\nfiltered_data = train_data.loc[:, train_data.isnull().sum() < threshold]\n\nnull_counts_filtered = filtered_data.isnull().sum()\n\n\nplt.figure(figsize=(15, 8))\nnull_counts_filtered.plot(kind='bar', color='blue')  \nplt.title('Null Values per Column (Filtered Data)', fontsize=16)\nplt.xlabel('Columns', fontsize=12)\nplt.ylabel('Number of Null Values', fontsize=12)\n\n\nplt.xticks(\n    range(len(null_counts_filtered)),  \n    [label if i % 10 == 0 else '' for i, label in enumerate(null_counts_filtered.index)],  \n    rotation=90,\n    ha='center',\n    fontsize=10\n)\n\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:26.149018Z","iopub.execute_input":"2024-11-19T16:04:26.149401Z","iopub.status.idle":"2024-11-19T16:04:27.971171Z","shell.execute_reply.started":"2024-11-19T16:04:26.149361Z","shell.execute_reply":"2024-11-19T16:04:27.969522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cleaned_unfiltered_data = train_data.dropna()\n\nprint(cleaned_unfiltered_data)\n\nprint(cleaned_unfiltered_data.info())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:27.972808Z","iopub.execute_input":"2024-11-19T16:04:27.973180Z","iopub.status.idle":"2024-11-19T16:04:28.502665Z","shell.execute_reply.started":"2024-11-19T16:04:27.973137Z","shell.execute_reply":"2024-11-19T16:04:28.501355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cleaned_train_data = filtered_train_data.dropna()\n\nprint(cleaned_train_data)\n\nprint(cleaned_train_data.info())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:28.504113Z","iopub.execute_input":"2024-11-19T16:04:28.504447Z","iopub.status.idle":"2024-11-19T16:04:28.656313Z","shell.execute_reply.started":"2024-11-19T16:04:28.504413Z","shell.execute_reply":"2024-11-19T16:04:28.654912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:28.657944Z","iopub.execute_input":"2024-11-19T16:04:28.658290Z","iopub.status.idle":"2024-11-19T16:04:28.664362Z","shell.execute_reply.started":"2024-11-19T16:04:28.658257Z","shell.execute_reply":"2024-11-19T16:04:28.663060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_data.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:28.665902Z","iopub.execute_input":"2024-11-19T16:04:28.666286Z","iopub.status.idle":"2024-11-19T16:04:28.679286Z","shell.execute_reply.started":"2024-11-19T16:04:28.666255Z","shell.execute_reply":"2024-11-19T16:04:28.677826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 0.2 * len(train_data)\n\ncolumns_to_impute = train_data.loc[:, train_data.isnull().sum() < threshold]\n\nnumeric_columns_to_impute = columns_to_impute.select_dtypes(include=[\"number\"])\n\nimputed_train_data = train_data.copy()\nfor column in numeric_columns_to_impute.columns:\n    imputed_train_data[column] = imputed_train_data[column].fillna(imputed_train_data[column].mean())\n\nprint(imputed_train_data)\n\nprint(imputed_train_data.isnull().sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:28.681032Z","iopub.execute_input":"2024-11-19T16:04:28.681445Z","iopub.status.idle":"2024-11-19T16:04:30.798360Z","shell.execute_reply.started":"2024-11-19T16:04:28.681412Z","shell.execute_reply":"2024-11-19T16:04:30.796907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_elements = imputed_train_data.size\n\ntotal_nan = imputed_train_data.isnull().sum().sum()\n\nnan_percentage = (total_nan / total_elements) * 100\n\nprint(f\"Percentage of NaN values in the imputed dataset: {nan_percentage:.2f}%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:30.800258Z","iopub.execute_input":"2024-11-19T16:04:30.800785Z","iopub.status.idle":"2024-11-19T16:04:31.347516Z","shell.execute_reply.started":"2024-11-19T16:04:30.800732Z","shell.execute_reply":"2024-11-19T16:04:31.346316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Statistics Before Imputation:\")\nprint(train_data.describe())\n\nprint(\"\\nStatistics After Imputation:\")\nprint(imputed_train_data.describe())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:31.348821Z","iopub.execute_input":"2024-11-19T16:04:31.349147Z","iopub.status.idle":"2024-11-19T16:04:40.914712Z","shell.execute_reply.started":"2024-11-19T16:04:31.349116Z","shell.execute_reply":"2024-11-19T16:04:40.913191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_data.isnull().sum())  \nprint(train_data.isnull().sum().sum())  \nprint(train_data.shape)  \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:40.916544Z","iopub.execute_input":"2024-11-19T16:04:40.917008Z","iopub.status.idle":"2024-11-19T16:04:41.971999Z","shell.execute_reply.started":"2024-11-19T16:04:40.916971Z","shell.execute_reply":"2024-11-19T16:04:41.970345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 0.8 * len(train_data)  \ntrain_data = train_data.dropna(axis=1, thresh=threshold)\n\nprint(train_data.shape) \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:41.973778Z","iopub.execute_input":"2024-11-19T16:04:41.974201Z","iopub.status.idle":"2024-11-19T16:04:42.706837Z","shell.execute_reply.started":"2024-11-19T16:04:41.974163Z","shell.execute_reply":"2024-11-19T16:04:42.705407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import necessary libraries\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import accuracy_score\nfrom xgboost import XGBRegressor\n\n# Load the dataset\nfile_path = '/kaggle/input/brist1d/train.csv'  # Path to your dataset\ndata = pd.read_csv(file_path)\n\n# Identify target and irrelevant columns\ntarget_column = 'bg+1:00'  # Adjust this to your actual target column\nirrelevant_columns = ['id', 'p_num', 'time']  # Adjust as needed\nX = data.drop(columns=[target_column] + irrelevant_columns)\ny = data[target_column]\n\n# Separate numeric and categorical columns\nnumeric_cols = X.select_dtypes(include=['number']).columns\ncategorical_cols = X.select_dtypes(exclude=['number']).columns\n\n# Handle missing values\nnumeric_imputer = SimpleImputer(strategy='mean')\ncategorical_imputer = SimpleImputer(strategy='most_frequent')\n\nX[numeric_cols] = numeric_imputer.fit_transform(X[numeric_cols])\nX[categorical_cols] = categorical_imputer.fit_transform(X[categorical_cols])\n\n# Encode categorical data\nlabel_encoders = {}\nfor col in categorical_cols:\n    le = LabelEncoder()\n    X[col] = le.fit_transform(X[col])\n    label_encoders[col] = le\n\n# Split the data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Train an XGBoost model\nxgb_model = XGBRegressor()  # Use XGBClassifier for classification tasks\nxgb_model.fit(X_train, y_train)\n\n# Make predictions on the test set\ny_pred = xgb_model.predict(X_test)\n\n# Evaluate model performance\nrmse = np.sqrt(np.mean((y_test - y_pred) ** 2))\nprint(f\"Root Mean Squared Error (RMSE): {rmse:.2f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:04:42.720101Z","iopub.execute_input":"2024-11-19T16:04:42.720491Z","iopub.status.idle":"2024-11-19T16:05:26.196417Z","shell.execute_reply.started":"2024-11-19T16:04:42.720423Z","shell.execute_reply":"2024-11-19T16:05:26.195334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"baseline_pred = [y.mean()] * len(y_test)\nbaseline_rmse = np.sqrt(np.mean((y_test - baseline_pred) ** 2))\nprint(f\"Baseline RMSE: {baseline_rmse:.2f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:05:26.197786Z","iopub.execute_input":"2024-11-19T16:05:26.198114Z","iopub.status.idle":"2024-11-19T16:05:26.217350Z","shell.execute_reply.started":"2024-11-19T16:05:26.198080Z","shell.execute_reply":"2024-11-19T16:05:26.215883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_target = y.mean()\naccuracy = (1 - (rmse / mean_target)) * 100\nprint(f\"Accuracy: {accuracy:.2f}%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T16:05:26.219166Z","iopub.execute_input":"2024-11-19T16:05:26.219692Z","iopub.status.idle":"2024-11-19T16:05:26.235164Z","shell.execute_reply.started":"2024-11-19T16:05:26.219617Z","shell.execute_reply":"2024-11-19T16:05:26.233740Z"}},"outputs":[],"execution_count":null}]}