{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"},{"sourceId":9662078,"sourceType":"datasetVersion","datasetId":5903284},{"sourceId":9664117,"sourceType":"datasetVersion","datasetId":5904708},{"sourceId":139850,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":118433,"modelId":141672},{"sourceId":140031,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":118583,"modelId":141836}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport gzip\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.impute import SimpleImputer\n\nimport torch\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch.optim import AdamW\nfrom torch.cuda.amp import GradScaler, autocast","metadata":{"execution":{"iopub.status.busy":"2024-10-19T03:51:57.103978Z","iopub.execute_input":"2024-10-19T03:51:57.104343Z","iopub.status.idle":"2024-10-19T03:51:57.111101Z","shell.execute_reply.started":"2024-10-19T03:51:57.104309Z","shell.execute_reply":"2024-10-19T03:51:57.110053Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.backends.cudnn.benchmark = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T03:51:59.595248Z","iopub.execute_input":"2024-10-19T03:51:59.595622Z","iopub.status.idle":"2024-10-19T03:51:59.600120Z","shell.execute_reply.started":"2024-10-19T03:51:59.595589Z","shell.execute_reply":"2024-10-19T03:51:59.599135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/brist1d-processed-normal-imputing/train_df_processed.csv\", low_memory=False)\ntest_df = pd.read_csv(\"/kaggle/input/brist1d-processed-normal-imputing/test_df_processed.csv\", low_memory=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T05:37:20.660505Z","iopub.execute_input":"2024-10-19T05:37:20.660887Z","iopub.status.idle":"2024-10-19T05:37:51.581990Z","shell.execute_reply.started":"2024-10-19T05:37:20.660851Z","shell.execute_reply":"2024-10-19T05:37:51.581091Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert time to Hour, Minute, Second\n# def convert_time_to_hms(time_series):\n#     hours = time_series.dt.hour\n#     minutes = time_series.dt.minute\n#     seconds = time_series.dt.second\n#     return pd.DataFrame({'hours': hours, 'minutes': minutes, 'seconds': seconds})","metadata":{"execution":{"iopub.status.busy":"2024-10-19T05:09:15.792203Z","iopub.execute_input":"2024-10-19T05:09:15.792577Z","iopub.status.idle":"2024-10-19T05:09:15.798050Z","shell.execute_reply.started":"2024-10-19T05:09:15.792537Z","shell.execute_reply":"2024-10-19T05:09:15.797108Z"},"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df['time'] = pd.to_datetime(train_df['time'], format='%H:%M:%S', errors='coerce')\n# test_df['time'] = pd.to_datetime(test_df['time'], format='%H:%M:%S', errors='coerce')","metadata":{"execution":{"iopub.status.busy":"2024-10-19T05:09:15.799326Z","iopub.execute_input":"2024-10-19T05:09:15.799660Z","iopub.status.idle":"2024-10-19T05:09:15.848553Z","shell.execute_reply.started":"2024-10-19T05:09:15.799626Z","shell.execute_reply":"2024-10-19T05:09:15.847630Z"},"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df = pd.concat([train_df, convert_time_to_hms(train_df['time'])], axis=1)\n# train_df = train_df.drop('time', axis=1)\n\n# test_df = pd.concat([test_df, convert_time_to_hms(test_df['time'])], axis=1)\n# test_df = test_df.drop('time', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T05:09:15.851222Z","iopub.execute_input":"2024-10-19T05:09:15.851883Z","iopub.status.idle":"2024-10-19T05:09:17.045732Z","shell.execute_reply.started":"2024-10-19T05:09:15.851835Z","shell.execute_reply":"2024-10-19T05:09:17.044713Z"},"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df = train_df.drop(['id', 'p_num'], axis=1)\n# test_df = test_df.drop(['id', 'p_num' ], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T05:09:17.047063Z","iopub.execute_input":"2024-10-19T05:09:17.047963Z","iopub.status.idle":"2024-10-19T05:09:17.411359Z","shell.execute_reply.started":"2024-10-19T05:09:17.047917Z","shell.execute_reply":"2024-10-19T05:09:17.410432Z"},"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Step 1: Identify common numerical columns between train_df and test_df\n# numerical_cols = train_df.select_dtypes(include=['float64', 'int64']).columns\n# common_numerical_cols = numerical_cols.intersection(test_df.columns)  # Select only the common columns\n\n# # Step 2: Fill missing values for the common numerical columns\n# train_df[common_numerical_cols] = train_df[common_numerical_cols].fillna(train_df[common_numerical_cols].mean())\n# test_df[common_numerical_cols] = test_df[common_numerical_cols].fillna(test_df[common_numerical_cols].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:09:17.412691Z","iopub.execute_input":"2024-10-19T05:09:17.413072Z","iopub.status.idle":"2024-10-19T05:09:20.140153Z","shell.execute_reply.started":"2024-10-19T05:09:17.413028Z","shell.execute_reply":"2024-10-19T05:09:20.139150Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Step 3: Handle categorical columns\n# categorical_cols = train_df.select_dtypes(include=['object']).columns  # Identify categorical columns\n\n# # Label encoding for categorical columns\n# for col in categorical_cols:\n#     train_df[col] = train_df[col].fillna('None')\n#     test_df[col] = test_df[col].fillna('None')\n    \n#     le = LabelEncoder()\n#     combined_data = pd.concat([train_df[col], test_df[col]], axis=0)  # Combine train and test to fit LabelEncoder\n#     le.fit(combined_data)\n    \n#     train_df[col] = le.transform(train_df[col])\n#     test_df[col] = le.transform(test_df[col])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:09:20.141690Z","iopub.execute_input":"2024-10-19T05:09:20.142094Z","iopub.status.idle":"2024-10-19T05:09:24.069024Z","shell.execute_reply.started":"2024-10-19T05:09:20.142048Z","shell.execute_reply":"2024-10-19T05:09:24.068242Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Save the processed train DataFrame to CSV\n# train_df.to_csv('train_df_processed.csv', index=False)\n# print(\"Processed train data saved as train_df_processed.csv\")\n\n# # Save the processed test DataFrame to CSV\n# test_df.to_csv('test_df_processed.csv', index=False)\n# print(\"Processed test data saved as test_df_processed.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:21:05.083413Z","iopub.execute_input":"2024-10-19T05:21:05.084085Z","iopub.status.idle":"2024-10-19T05:22:54.623828Z","shell.execute_reply.started":"2024-10-19T05:21:05.084046Z","shell.execute_reply":"2024-10-19T05:22:54.622899Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# VAE","metadata":{}},{"cell_type":"code","source":"# # Check if GPU is available\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# print(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:51:43.694952Z","iopub.execute_input":"2024-10-18T18:51:43.695577Z","iopub.status.idle":"2024-10-18T18:51:43.734956Z","shell.execute_reply.started":"2024-10-18T18:51:43.695534Z","shell.execute_reply":"2024-10-18T18:51:43.733615Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_train_tensor = torch.tensor(X_scaled, dtype=torch.float32).to(device)\n# test_tensor = torch.tensor(test_df_scaled, dtype=torch.float32).to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:51:44.258678Z","iopub.execute_input":"2024-10-18T18:51:44.259063Z","iopub.status.idle":"2024-10-18T18:51:44.608929Z","shell.execute_reply.started":"2024-10-18T18:51:44.259026Z","shell.execute_reply":"2024-10-18T18:51:44.607901Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Replace the missing values with VAE\n\n# class VAE(nn.Module):\n#     def __init__(self, input_dim, latent_dim, hidden_dim):\n#         super(VAE, self).__init__()\n#         # Encoder\n#         self.fc1 = nn.Linear(input_dim, hidden_dim)\n#         self.fc_mu = nn.Linear(hidden_dim, latent_dim)  # Latent mean\n#         self.fc_logvar = nn.Linear(hidden_dim, latent_dim)  # Latent log-variance\n        \n#         # Decoder\n#         self.fc3 = nn.Linear(latent_dim, hidden_dim)\n#         self.fc4 = nn.Linear(hidden_dim, input_dim)\n\n#     def encode(self, x):\n#         h = torch.relu(self.fc1(x))\n#         mu = self.fc_mu(h)\n#         logvar = self.fc_logvar(h)\n#         return mu, logvar\n\n#     def reparameterize(self, mu, logvar):\n#         std = torch.exp(0.5 * logvar)\n#         eps = torch.randn_like(std)\n#         return mu + eps * std\n\n#     def decode(self, z):\n#         h = torch.relu(self.fc3(z))\n#         return self.fc4(h)\n\n#     def forward(self, x):\n#         mu, logvar = self.encode(x)\n#         z = self.reparameterize(mu, logvar)\n#         return self.decode(z), mu, logvar","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:51:46.228207Z","iopub.execute_input":"2024-10-18T18:51:46.228587Z","iopub.status.idle":"2024-10-18T18:51:46.238391Z","shell.execute_reply.started":"2024-10-18T18:51:46.228549Z","shell.execute_reply":"2024-10-18T18:51:46.237496Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Define model parameters\n# input_dim = 432  # Number of features\n# latent_dim = 2\n# hidden_dim = 64\n\n# # Recreate the VAE model structure\n# vae = VAE(input_dim, latent_dim, hidden_dim).to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:53:12.315576Z","iopub.execute_input":"2024-10-18T18:53:12.315953Z","iopub.status.idle":"2024-10-18T18:53:12.323808Z","shell.execute_reply.started":"2024-10-18T18:53:12.315917Z","shell.execute_reply":"2024-10-18T18:53:12.322794Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Define the file path\n# model_save_path = '/kaggle/working/vae_model.pth'\n\n# # Ask user if they want to delete the old model file\n# delete_old = input(\"Do you want to delete the old model file and start a new training? (yes/no): \").strip().lower()\n\n# if delete_old == 'yes':\n#     # Check if the file exists, and delete it if it does\n#     if os.path.exists(model_save_path):\n#         os.remove(model_save_path)\n#         print(f\"Deleted old model file: {model_save_path}\")\n#     else:\n#         print(f\"File not found: {model_save_path}\")\n# else:\n#     print(\"Old model file not deleted, proceeding with training...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:52:10.476716Z","iopub.execute_input":"2024-10-18T18:52:10.477099Z","iopub.status.idle":"2024-10-18T18:52:13.032670Z","shell.execute_reply.started":"2024-10-18T18:52:10.477060Z","shell.execute_reply":"2024-10-18T18:52:13.031693Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(f\"X_train_tensor shape: {X_train_tensor.shape}\")  # Should output [177024, 504]\n# print(f\"test_tensor shape: {test_tensor.shape}\")        # Should output [3644, 504]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:53:15.872183Z","iopub.execute_input":"2024-10-18T18:53:15.872551Z","iopub.status.idle":"2024-10-18T18:53:15.877889Z","shell.execute_reply.started":"2024-10-18T18:53:15.872517Z","shell.execute_reply":"2024-10-18T18:53:15.876920Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# vae = VAE(input_dim, latent_dim, hidden_dim).to(device)\n# optimizer = optim.Adam(vae.parameters(), lr=0.001)\n\n# # Define the VAE loss function (reconstruction + KL divergence)\n# def vae_loss(recon_x, x, mu, logvar):\n#     # Reconstruction loss (Mean Squared Error)\n#     recon_loss = nn.MSELoss()(recon_x, x)\n#     # KL divergence\n#     kl_loss = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n#     return recon_loss + kl_loss\n\n# # Define EarlyStopping class\n# class EarlyStopping:\n#     def __init__(self, patience=10, delta=0.001):\n#         self.patience = patience\n#         self.counter = 0\n#         self.best_loss = None\n#         self.early_stop = False\n#         self.delta = delta\n\n#     def __call__(self, val_loss):\n#         if self.best_loss is None:\n#             self.best_loss = val_loss\n#         elif val_loss > self.best_loss - self.delta:\n#             self.counter += 1\n#             if self.counter >= self.patience:\n#                 self.early_stop = True\n#         else:\n#             self.best_loss = val_loss\n#             self.counter = 0\n\n# # Set up early stopping\n# early_stopping = EarlyStopping(patience=10)\n\n# # Add learning rate scheduler (ReduceLROnPlateau)\n# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n\n# # Train the VAE with progress bar and early stopping\n# epochs = 10000  # Increase the number of epochs if needed\n# best_loss = float('inf')\n\n# # Initialize the progress bar\n# pbar = tqdm(range(epochs), desc=\"Training VAE\")\n\n# # Training loop\n# for epoch in pbar:\n#     vae.train()\n#     optimizer.zero_grad()\n    \n#     # Forward pass\n#     recon_data, mu, logvar = vae(X_train_tensor)\n    \n#     # Compute loss\n#     loss = vae_loss(recon_data, X_train_tensor, mu, logvar)\n    \n#     # Backward pass and optimization\n#     loss.backward()\n#     optimizer.step()\n\n#     # Update progress bar with loss inline\n#     pbar.set_postfix({\"loss\": loss.item()})\n\n#     # Reduce learning rate on plateau (based on loss)\n#     scheduler.step(loss.item())\n    \n#     # Check and print learning rate if it has changed\n#     if epoch > 0 and scheduler.optimizer.param_groups[0]['lr'] != scheduler.get_last_lr()[0]:\n#         pbar.write(f\"Learning rate reduced to {scheduler.get_last_lr()[0]}\")\n\n#     # Early stopping logic\n#     early_stopping(loss.item())\n#     if early_stopping.early_stop:\n#         pbar.set_postfix({\"best_loss\": early_stopping.best_loss})\n#         pbar.write(f\"Early stopping at epoch {epoch}. Best loss: {early_stopping.best_loss}\")\n#         break\n\n# # After training, save the model (overwrite if it exists)\n# torch.save(vae.state_dict(), model_save_path)\n# print(f\"Model saved to {model_save_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:53:23.417174Z","iopub.execute_input":"2024-10-18T18:53:23.417547Z","iopub.status.idle":"2024-10-18T18:57:25.596911Z","shell.execute_reply.started":"2024-10-18T18:53:23.417512Z","shell.execute_reply":"2024-10-18T18:57:25.595926Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Switch to evaluation mode\n# vae.eval()\n\n# # Perform imputation on the original train and test data with missing values\n# with torch.no_grad():\n#     # Impute training data\n#     imputed_train_data, _, _ = vae(X_train_tensor)\n\n#     # Similarly, impute the test data\n#     imputed_test_data, _, _ = vae(test_tensor)\n\n# # Ensure alignment of common numerical columns between train and test sets\n# common_numerical_cols = train_df.select_dtypes(include=['float64', 'int64']).columns.intersection(test_df.columns)\n\n# # Move the imputed data back to CPU and convert to NumPy\n# imputed_train_data_np = imputed_train_data.cpu().numpy()\n# imputed_test_data_np = imputed_test_data.cpu().numpy()\n\n# # Inverse transform the scaled data (apply scaler only on numerical columns)\n# imputed_train_rescaled = scaler.inverse_transform(imputed_train_data_np)\n# imputed_test_rescaled = scaler.inverse_transform(imputed_test_data_np)\n\n# # Extract the first 432 columns from common_numerical_cols (the columns used by the VAE)\n# vae_numerical_cols = common_numerical_cols[:432]\n\n# # Rebuild the missing masks to only include these 432 columns\n# missing_mask_train = train_df[vae_numerical_cols].isnull()\n# missing_mask_test = test_df[vae_numerical_cols].isnull()\n\n# # Now, replace the missing values in both train and test DataFrames for the 432 VAE columns\n# train_data_imputed = train_df.copy()\n# test_data_imputed = test_df.copy()\n\n# # Replace missing values in the 432 columns used by the VAE\n# for i, col in enumerate(vae_numerical_cols):\n#     # Update only the missing values for each column in both train and test\n#     train_data_imputed.loc[missing_mask_train[col], col] = imputed_train_rescaled[:, i]\n#     test_data_imputed.loc[missing_mask_test[col], col] = imputed_test_rescaled[:, i]\n\n# # View the result\n# print(\"Train Data Imputed: \\n\", train_data_imputed[vae_numerical_cols].head())\n# print(\"Test Data Imputed: \\n\", test_data_imputed[vae_numerical_cols].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-18T18:57:25.598585Z","iopub.execute_input":"2024-10-18T18:57:25.599095Z","iopub.status.idle":"2024-10-18T18:57:28.517555Z","shell.execute_reply.started":"2024-10-18T18:57:25.599058Z","shell.execute_reply":"2024-10-18T18:57:28.516523Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# GARNN","metadata":{}},{"cell_type":"code","source":"# train_df = pd.read_csv(\"/kaggle/input/brist1d-blood-glucose-vaes-imputing/train_data_imputed.csv\", low_memory=False)\n# test_df = pd.read_csv(\"/kaggle/input/brist1d-blood-glucose-vaes-imputing/test_data_imputed.csv\", low_memory=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T03:43:30.826580Z","iopub.execute_input":"2024-10-19T03:43:30.827307Z","iopub.status.idle":"2024-10-19T03:44:01.798504Z","shell.execute_reply.started":"2024-10-19T03:43:30.827268Z","shell.execute_reply":"2024-10-19T03:44:01.797683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check if GPU is available and set the device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Using device: {device}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T03:44:01.800260Z","iopub.execute_input":"2024-10-19T03:44:01.800795Z","iopub.status.idle":"2024-10-19T03:44:01.846746Z","shell.execute_reply.started":"2024-10-19T03:44:01.800746Z","shell.execute_reply":"2024-10-19T03:44:01.845639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the target column\ntarget = 'bg+1:00'\n\n# Separate the features (X) and target (y) from the training dataset\nX_train = train_df.drop(columns=[target])\ny_train = train_df[target]\n\nX_test = test_df\n\n# Scale the features (StandardScaler)\nscaler = StandardScaler()\n\n# Fit the scaler on the training data and transform both train and test\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\n# Convert the data to PyTorch tensors for use in the GARNN model\nX_train_tensor = torch.tensor(X_train_scaled, dtype=torch.float32).unsqueeze(1).contiguous().to(device)\ny_train_tensor = torch.tensor(y_train.values, dtype=torch.float32).to(device)\nX_test_tensor = torch.tensor(X_test_scaled, dtype=torch.float32).unsqueeze(1).contiguous().to(device)\n\n# Check the shapes of tensors\nprint(f\"X_train_tensor shape: {X_train_tensor.shape}\")\nprint(f\"y_train_tensor shape: {y_train_tensor.shape}\")\nprint(f\"X_test_tensor shape: {X_test_tensor.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:12:59.931952Z","iopub.execute_input":"2024-10-19T05:12:59.932762Z","iopub.status.idle":"2024-10-19T05:13:02.643965Z","shell.execute_reply.started":"2024-10-19T05:12:59.932705Z","shell.execute_reply":"2024-10-19T05:13:02.642934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_dataset = TensorDataset(X_train_tensor, y_train_tensor)\n# train_loader = DataLoader(train_dataset, batch_size=512, shuffle=True)  # Reduce batch size","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T03:44:21.771283Z","iopub.execute_input":"2024-10-19T03:44:21.772137Z","iopub.status.idle":"2024-10-19T03:44:21.775956Z","shell.execute_reply.started":"2024-10-19T03:44:21.772096Z","shell.execute_reply":"2024-10-19T03:44:21.774860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GraphAttentionLayer(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(GraphAttentionLayer, self).__init__()\n        self.fc = nn.Linear(input_dim, output_dim)\n        self.attention = nn.Parameter(torch.FloatTensor(output_dim, 1))\n        nn.init.xavier_uniform_(self.attention)\n\n    def forward(self, x):\n        h = torch.relu(self.fc(x))\n        attention_scores = torch.matmul(h, self.attention).squeeze(-1)\n        attention_weights = torch.softmax(attention_scores, dim=-1)\n        h_prime = h * attention_weights.unsqueeze(-1)\n        return h_prime\n\nclass GARNN(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim, graph_out_dim, num_layers=1, dropout_rate=0.3):\n        super(GARNN, self).__init__()\n        self.graph_attention = GraphAttentionLayer(input_dim, graph_out_dim)\n        self.gru = nn.GRU(graph_out_dim, hidden_dim, num_layers, batch_first=True)\n        self.dropout = nn.Dropout(p=dropout_rate)\n        self.fc = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        x = self.graph_attention(x)\n        x = x.contiguous()\n        out, _ = self.gru(x)\n        out = self.dropout(out[:, -1, :])\n        out = self.fc(out)\n        return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:13:46.118644Z","iopub.execute_input":"2024-10-19T05:13:46.119307Z","iopub.status.idle":"2024-10-19T05:13:46.129605Z","shell.execute_reply.started":"2024-10-19T05:13:46.119259Z","shell.execute_reply":"2024-10-19T05:13:46.128654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters\ninput_dim = X_train_tensor.shape[2]  # Number of features\ngraph_out_dim = 64\nhidden_dim = 128\noutput_dim = 1\nnum_layers = 2\nlr = 0.001\nweight_decay = 1e-5\nepochs = 500\n\n# Define HuberLoss as the loss function\ncriterion = torch.nn.HuberLoss()\n\n# K-Fold Cross-Validation\nkf = KFold(n_splits=5)\n\n# Initialize the model (assuming GARNN is defined elsewhere)\nmodel = GARNN(input_dim, hidden_dim, output_dim, graph_out_dim, num_layers).to(device)  # Move model to GPU/CPU\n\n# Define the optimizer\noptimizer = AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n\n# Mixed precision scaler\nscaler = torch.amp.GradScaler(\"cuda\")\n\n# EarlyStopping class\nclass EarlyStopping:\n    def __init__(self, patience=10, delta=0.001):\n        self.patience = patience\n        self.counter = 0\n        self.best_loss = None\n        self.early_stop = False\n        self.delta = delta\n\n    def __call__(self, val_loss):\n        if self.best_loss is None:\n            self.best_loss = val_loss\n        elif val_loss > self.best_loss - self.delta:\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_loss = val_loss\n            self.counter = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:13:49.957160Z","iopub.execute_input":"2024-10-19T05:13:49.957923Z","iopub.status.idle":"2024-10-19T05:13:49.974243Z","shell.execute_reply.started":"2024-10-19T05:13:49.957879Z","shell.execute_reply":"2024-10-19T05:13:49.973190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Training loop with K-Fold Cross-Validation\n# for fold, (train_idx, val_idx) in enumerate(kf.split(X_train_tensor)):\n#     print(f\"Fold {fold+1}\")\n\n#     # Split into training and validation sets for this fold\n#     X_train_fold, X_val_fold = X_train_tensor[train_idx], X_train_tensor[val_idx]\n#     y_train_fold, y_val_fold = y_train_tensor[train_idx], y_train_tensor[val_idx]\n\n#     # Create DataLoaders for batch processing\n#     train_dataset = TensorDataset(X_train_fold, y_train_fold)\n#     val_dataset = TensorDataset(X_val_fold, y_val_fold)\n\n#     train_loader = DataLoader(train_dataset, batch_size=1024, shuffle=True)\n#     val_loader = DataLoader(val_dataset, batch_size=1024)\n\n#     # Initialize the model, optimizer, and scheduler for each fold\n#     model = GARNN(input_dim, hidden_dim, output_dim, graph_out_dim, num_layers).to(device)\n#     optimizer = AdamW(model.parameters(), lr=0.001, weight_decay=1e-5)\n    \n#     # Initialize EarlyStopping\n#     early_stopping = EarlyStopping(patience=10, delta=0.001)\n\n#     # Training loop\n#     pbar = tqdm(range(epochs), desc=f\"Training Fold {fold+1}\", leave=False)\n#     for epoch in pbar:\n#         model.train()\n#         running_loss = 0.0\n\n#         for X_batch, y_batch in train_loader:\n#             X_batch, y_batch = X_batch.to(device, non_blocking=True), y_batch.to(device, non_blocking=True)\n#             optimizer.zero_grad()\n\n#             # Use mixed precision training for faster performance\n#             with torch.amp.autocast(device_type=device):\n#                 # Forward pass\n#                 output = model(X_batch)\n                \n#                 # Ensure y_batch has the correct shape\n#                 y_batch = y_batch.view(-1, 1)\n\n#                 # Compute loss\n#                 loss = criterion(output, y_batch)\n#                 running_loss += loss.item() * X_batch.size(0)  # Sum loss for batch\n\n#             # Backward pass with scaler for mixed precision\n#             scaler.scale(loss).backward()\n#             scaler.step(optimizer)\n#             scaler.update()\n\n#         # Validation loop\n#         model.eval()\n#         val_loss = 0.0\n#         with torch.no_grad():\n#             for X_batch, y_batch in val_loader:\n#                 X_batch, y_batch = X_batch.to(device, non_blocking=True), y_batch.to(device, non_blocking=True)\n\n#                 # Forward pass\n#                 output = model(X_batch)\n#                 y_batch = y_batch.view(-1, 1)\n\n#                 # Compute validation loss\n#                 loss = criterion(output, y_batch)\n#                 val_loss += loss.item() * X_batch.size(0)\n\n#         # Calculate average losses for training and validation\n#         train_loss = running_loss / len(train_loader.dataset)\n#         val_loss = val_loss / len(val_loader.dataset)\n\n#         # Use the progress bar's `set_postfix` method to display the loss inline\n#         pbar.set_postfix({\"Train Loss\": f\"{train_loss:.4f}\", \"Val Loss\": f\"{val_loss:.4f}\"})\n        \n#         # Early stopping logic\n#         early_stopping(val_loss)\n#         if early_stopping.early_stop:\n#             print(f\"\\nEarly stopping at epoch {epoch+1}. Best Validation Loss: {early_stopping.best_loss:.4f}\")\n#             break\n\n#     print(f\"\\nFold {fold+1} completed. Final Validation Loss: {val_loss:.4f}\")\n\n#     # Clear GPU cache after each fold\n#     torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:14:40.996073Z","iopub.execute_input":"2024-10-19T05:14:40.996797Z","iopub.status.idle":"2024-10-19T05:18:37.062431Z","shell.execute_reply.started":"2024-10-19T05:14:40.996753Z","shell.execute_reply":"2024-10-19T05:18:37.061534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_save_path = \"/kaggle/working/final_garnn_model.pth\"\n\n# # Save the final model (this will overwrite the model if it already exists)\n# torch.save(model.state_dict(), model_save_path)\n# print(f\"Final model saved to {model_save_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:19:18.148071Z","iopub.execute_input":"2024-10-19T05:19:18.148464Z","iopub.status.idle":"2024-10-19T05:19:18.158694Z","shell.execute_reply.started":"2024-10-19T05:19:18.148427Z","shell.execute_reply":"2024-10-19T05:19:18.157684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_save_path = \"/kaggle/input/garnn-model/pytorch/default/1/final_garnn_model.pth\"\n\n# Load the saved model function\ndef load_model(model_save_path, model):\n    if os.path.exists(model_save_path):\n        model.load_state_dict(torch.load(model_save_path))\n        print(f\"Model loaded from {model_save_path}\")\n    else:\n        print(f\"Model file {model_save_path} not found.\")\n        return None\n    return model\n    \n# Initialize the model (ensure you use the same architecture as during training)\nmodel = GARNN(input_dim, hidden_dim, output_dim, graph_out_dim, num_layers).to(device)\n\n# Load the saved model\nmodel = load_model(model_save_path, model)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\n\nwith torch.no_grad():\n    final_predictions = model(X_test_tensor).cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:19:28.224506Z","iopub.execute_input":"2024-10-19T05:19:28.224891Z","iopub.status.idle":"2024-10-19T05:19:28.241091Z","shell.execute_reply.started":"2024-10-19T05:19:28.224853Z","shell.execute_reply":"2024-10-19T05:19:28.240311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_predictions[:10]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:19:43.593089Z","iopub.execute_input":"2024-10-19T05:19:43.593512Z","iopub.status.idle":"2024-10-19T05:19:43.600424Z","shell.execute_reply.started":"2024-10-19T05:19:43.593472Z","shell.execute_reply":"2024-10-19T05:19:43.599302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/brist1d/sample_submission.csv')\n\nif final_predictions.shape[1] == 1:\n    final_predictions = final_predictions.flatten()  # Convert to 1D array if necessary\n\nsample_submission['bg+1:00'] = final_predictions\nsample_submission.to_csv('submission.csv', index=False)\nprint(\"Predictions saved to submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-19T05:05:36.864387Z","iopub.execute_input":"2024-10-19T05:05:36.865293Z","iopub.status.idle":"2024-10-19T05:05:36.897132Z","shell.execute_reply.started":"2024-10-19T05:05:36.865248Z","shell.execute_reply":"2024-10-19T05:05:36.896259Z"}},"outputs":[],"execution_count":null}]}