{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install torch-geometric and its dependencies\n!pip install torch-scatter torch-sparse torch-geometric -f https://data.pyg.org/whl/torch-2.0.0+cpu.html","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:41:28.458826Z","iopub.execute_input":"2024-11-05T15:41:28.459700Z","iopub.status.idle":"2024-11-05T15:41:41.278326Z","shell.execute_reply.started":"2024-11-05T15:41:28.459657Z","shell.execute_reply":"2024-11-05T15:41:41.277255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:41:41.280846Z","iopub.execute_input":"2024-11-05T15:41:41.281918Z","iopub.status.idle":"2024-11-05T15:41:41.286893Z","shell.execute_reply.started":"2024-11-05T15:41:41.281860Z","shell.execute_reply":"2024-11-05T15:41:41.285938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch \ndevice = torch.device(\"cpu\")  # Set device to CPU","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:41:41.288328Z","iopub.execute_input":"2024-11-05T15:41:41.288750Z","iopub.status.idle":"2024-11-05T15:41:41.297788Z","shell.execute_reply.started":"2024-11-05T15:41:41.288705Z","shell.execute_reply":"2024-11-05T15:41:41.296960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cleaning the dataset and data preprocessing","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch_geometric.nn import GCNConv\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import roc_auc_score, log_loss\nimport gc\n\n\n# Load the dataset using polars for better efficiency\ntrain_schema = {\n    'id': pl.UInt64,\n    'click': pl.Int8,\n    'hour': pl.String,\n    'C1': pl.Int16,\n    'banner_pos': pl.Int8,\n    'site_id': pl.Categorical,\n    'site_domain': pl.Categorical,\n    'site_category': pl.Categorical,\n    'app_id': pl.Categorical,\n    'app_domain': pl.Categorical,\n    'app_category': pl.Categorical,\n    'device_id': pl.Categorical,\n    'device_ip': pl.Categorical,\n    'device_model': pl.Categorical,\n    'device_type': pl.Int8,\n    'device_conn_type': pl.Int8,\n    'C14': pl.Int16,\n    'C15': pl.Int16,\n    'C16': pl.Int16,\n    'C17': pl.Int16,\n    'C18': pl.Int16,\n    'C19': pl.Int16,\n    'C20': pl.Int32,\n    'C21': pl.Int16\n}\n\ntest_schema = {\n    'id': pl.UInt64,\n    'hour': pl.String,\n    'C1': pl.Int16,\n    'banner_pos': pl.Int8,\n    'site_id': pl.Categorical,\n    'site_domain': pl.Categorical,\n    'site_category': pl.Categorical,\n    'app_id': pl.Categorical,\n    'app_domain': pl.Categorical,\n    'app_category': pl.Categorical,\n    'device_id': pl.Categorical,\n    'device_ip': pl.Categorical,\n    'device_model': pl.Categorical,\n    'device_type': pl.Int8,\n    'device_conn_type': pl.Int8,\n    'C14': pl.Int16,\n    'C15': pl.Int16,\n    'C16': pl.Int16,\n    'C17': pl.Int16,\n    'C18': pl.Int16,\n    'C19': pl.Int16,\n    'C20': pl.Int32,\n    'C21': pl.Int16\n}\n\n# Load train and test datasets\ntrain_df = pl.read_csv('/kaggle/input/avazu-ctr-prediction/train.gz', schema=train_schema)\ntest_df = pl.read_csv('/kaggle/input/avazu-ctr-prediction/test.gz', schema=test_schema)\n\n# Step 1: Sample a much smaller fraction (0.1%) of the data to reduce size\ntrain_df_sampled = train_df.sample(n=int(len(train_df) * 0.0001))  # Sample 0.01%\ntest_df_sampled = test_df.sample(n=int(len(test_df) * 0.0001))      # Sample 0.01%\n\n# Step 2: Feature Engineering - Extract 'hour_of_day' and 'day_of_month' from 'hour'\ndef extract_hour_features(df):\n    df = df.with_columns([\n        (pl.col('hour').str.slice(6, 2).cast(pl.Int8)).alias('hour_of_day'),   # Extract hour\n        (pl.col('hour').str.slice(4, 2).cast(pl.Int8)).alias('day_of_month')   # Extract day\n    ])\n    return df.drop('hour')\n\ntrain_df_sampled = extract_hour_features(train_df_sampled)\ntest_df_sampled = extract_hour_features(test_df_sampled)\n\n# Step 3: Remove unnecessary columns - Keep only a subset of important columns\nimportant_columns = [\n    'click', 'C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n    'app_id', 'app_domain', 'app_category', 'device_id', 'device_model', \n    'device_type', 'device_conn_type', 'C14', 'C15', 'C16', 'hour_of_day', 'day_of_month'\n]\n\n# Select only these columns\ntrain_df_trimmed = train_df_sampled.select(important_columns)\ntest_df_trimmed = test_df_sampled.select([col for col in important_columns if col != 'click'])  # Exclude 'click' from test set\n\n# Step 4: Encode categorical features using LabelEncoder\ncategorical_columns = ['site_id', 'site_domain', 'site_category', 'app_id', 'app_domain',\n                       'app_category', 'device_id', 'device_model']\n\ndef label_encode_columns(df, columns):\n    for col in columns:\n        le = LabelEncoder()\n        df = df.with_columns([(pl.Series(col, le.fit_transform(df[col].to_numpy()))).alias(col)])\n    return df\n\n# Apply label encoding to train and test sets\ntrain_df_encoded = label_encode_columns(train_df_trimmed, categorical_columns)\ntest_df_encoded = label_encode_columns(test_df_trimmed, categorical_columns)\n\n# Step 5: Separate features and labels for training\nX_train = train_df_encoded.drop(['click']).to_pandas().values\ny_train = train_df_encoded['click'].to_pandas().values\n\n# Prepare Test Data\nX_test = test_df_encoded.to_pandas().values","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:41:41.300330Z","iopub.execute_input":"2024-11-05T15:41:41.300688Z","iopub.status.idle":"2024-11-05T15:42:36.707848Z","shell.execute_reply.started":"2024-11-05T15:41:41.300654Z","shell.execute_reply":"2024-11-05T15:42:36.706850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data into train and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:36.709211Z","iopub.execute_input":"2024-11-05T15:42:36.709669Z","iopub.status.idle":"2024-11-05T15:42:36.716812Z","shell.execute_reply.started":"2024-11-05T15:42:36.709610Z","shell.execute_reply":"2024-11-05T15:42:36.715728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Graph construction","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport torch\n\ndef create_sparse_graph(X, threshold=0.05):\n    num_nodes = X.shape[0]\n    \n    # Create an array of possible edges (i, j) where i < j\n    row_indices = np.repeat(np.arange(num_nodes), num_nodes - 1)\n    col_indices = np.concatenate([np.delete(np.arange(num_nodes), i) for i in range(num_nodes)])\n    \n    # Generate random values to determine if an edge is created\n    edge_probs = np.random.rand(len(row_indices))\n    \n    # Select edges where the random value is less than the threshold\n    valid_edges = edge_probs < threshold\n    \n    # Get the selected edges\n    edge_index = np.column_stack((row_indices[valid_edges], col_indices[valid_edges]))\n    \n    # Ensure valid edges\n    if edge_index.size == 0:  # If no edges were added\n        edge_index = np.array([[0, 1], [1, 0]])  # Default edge to avoid empty graph\n    \n    edge_index = torch.tensor(edge_index, dtype=torch.long).t()\n    \n    # Convert feature matrix to tensor\n    X_tensor = torch.tensor(X, dtype=torch.float)\n    \n    return X_tensor, edge_index","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:36.718276Z","iopub.execute_input":"2024-11-05T15:42:36.718634Z","iopub.status.idle":"2024-11-05T15:42:36.734534Z","shell.execute_reply.started":"2024-11-05T15:42:36.718589Z","shell.execute_reply":"2024-11-05T15:42:36.733479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_tensor, edge_index = create_sparse_graph(X_train, threshold=0.1)  # Increase the threshold to reduce edge density\nX_val_tensor, _ = create_sparse_graph(X_val, threshold=0.01)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:36.735830Z","iopub.execute_input":"2024-11-05T15:42:36.736210Z","iopub.status.idle":"2024-11-05T15:42:37.065721Z","shell.execute_reply.started":"2024-11-05T15:42:36.736173Z","shell.execute_reply":"2024-11-05T15:42:37.064612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GCN Model","metadata":{}},{"cell_type":"code","source":"class GCNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super(GCNModel, self).__init__()\n        self.conv1 = GCNConv(input_dim, hidden_dim)\n        self.conv2 = GCNConv(hidden_dim, output_dim)\n        \n    def forward(self, x, edge_index):\n        x = self.conv1(x, edge_index)\n        x = torch.relu(x)\n        x = self.conv2(x, edge_index)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:37.067087Z","iopub.execute_input":"2024-11-05T15:42:37.067425Z","iopub.status.idle":"2024-11-05T15:42:37.074207Z","shell.execute_reply.started":"2024-11-05T15:42:37.067388Z","shell.execute_reply":"2024-11-05T15:42:37.072967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_gcn_model(X_train, y_train, X_val, y_val, edge_index, input_dim, hidden_dim, output_dim, epochs=100, lr=0.01, batch_size=32):\n    model = GCNModel(input_dim, hidden_dim, output_dim).to(device)\n    criterion = nn.BCEWithLogitsLoss()  # Adjust based on your task (e.g., binary classification)\n    optimizer = optim.Adam(model.parameters(), lr=lr)\n\n    # Convert labels to tensor and move to the correct device\n    y_train_tensor = torch.tensor(y_train, dtype=torch.float).to(device)\n    X_train = X_train.to(device)\n    edge_index = edge_index.to(device)\n\n    # Training loop\n    for epoch in range(epochs):\n        model.train()\n        optimizer.zero_grad()\n\n        # Forward pass\n        out = model(X_train, edge_index)\n        \n        # Compute loss\n        loss = criterion(out.view(-1), y_train_tensor)\n        loss.backward()\n        optimizer.step()\n        \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}')\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:37.075605Z","iopub.execute_input":"2024-11-05T15:42:37.075973Z","iopub.status.idle":"2024-11-05T15:42:37.087275Z","shell.execute_reply.started":"2024-11-05T15:42:37.075934Z","shell.execute_reply":"2024-11-05T15:42:37.086368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"X_train_tensor shape: {X_train_tensor.shape}, dtype: {X_train_tensor.dtype}\")\nprint(f\"X_val_tensor shape: {X_val_tensor.shape}, dtype: {X_val_tensor.dtype}\")\nprint(f\"edge_index shape: {edge_index.shape}, dtype: {edge_index.dtype}\")\nprint(f\"y_train shape: {y_train.shape}, dtype: {y_train.dtype}\")\nprint(f\"Unique labels in y_train: {np.unique(y_train)}\")  # Check if labels are 0 or 1","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:37.090556Z","iopub.execute_input":"2024-11-05T15:42:37.091324Z","iopub.status.idle":"2024-11-05T15:42:37.103658Z","shell.execute_reply.started":"2024-11-05T15:42:37.091284Z","shell.execute_reply":"2024-11-05T15:42:37.102467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for NaN or invalid values in y_train\nprint(f\"y_train contains NaN: {np.isnan(y_train).any()}\")\nprint(f\"y_train contains values outside [0, 1]: {(y_train < 0).any() or (y_train > 1).any()}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:37.107470Z","iopub.execute_input":"2024-11-05T15:42:37.108135Z","iopub.status.idle":"2024-11-05T15:42:37.114511Z","shell.execute_reply.started":"2024-11-05T15:42:37.108084Z","shell.execute_reply":"2024-11-05T15:42:37.113400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Try converting to tensor without sending to device first\ny_train_tensor = torch.tensor(y_train, dtype=torch.float32)\n\n# Print tensor stats before sending to GPU\nprint(f\"y_train_tensor stats - Min: {y_train_tensor.min()}, Max: {y_train_tensor.max()}\")\n\n# Now send to device\ny_train_tensor = y_train_tensor.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:37.115835Z","iopub.execute_input":"2024-11-05T15:42:37.116228Z","iopub.status.idle":"2024-11-05T15:42:37.125145Z","shell.execute_reply.started":"2024-11-05T15:42:37.116184Z","shell.execute_reply":"2024-11-05T15:42:37.124144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert y_train to torch.float32 for BCEWithLogitsLoss\ny_train_tensor = torch.tensor(y_train, dtype=torch.float32).to(device)\n\n# Move tensors to device\nX_train_tensor = X_train_tensor.to(device)\nX_val_tensor = X_val_tensor.to(device)\nedge_index = edge_index.to(device)\n\n# Check after moving to device\n# Assuming the tensors are already defined and moved to CPU\nprint(f\"X_train_tensor is on device: {X_train_tensor.device}\")\nprint(f\"X_val_tensor is on device: {X_val_tensor.device}\")\nprint(f\"edge_index is on device: {edge_index.device}\")\nprint(f\"y_train_tensor is on device: {y_train_tensor.device}\")\n\n# Define input/output dimensions\ninput_dim = X_train_tensor.shape[1]\nhidden_dim = 64  # You can adjust this\noutput_dim = 1   # Adjust based on your task\n\n# Train the model\nmodel = train_gcn_model(X_train_tensor, y_train_tensor, X_val_tensor, y_val, edge_index, input_dim, hidden_dim, output_dim)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:42:37.126657Z","iopub.execute_input":"2024-11-05T15:42:37.127090Z","iopub.status.idle":"2024-11-05T15:43:44.865741Z","shell.execute_reply.started":"2024-11-05T15:42:37.127049Z","shell.execute_reply":"2024-11-05T15:43:44.864507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"X_val_tensor on device: {X_val_tensor.device}\")\nprint(f\"edge_index on device: {edge_index.device}\")\nprint(f\"y_val on device: {torch.tensor(y_val, dtype=torch.float32).device}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:43:44.867396Z","iopub.execute_input":"2024-11-05T15:43:44.868521Z","iopub.status.idle":"2024-11-05T15:43:44.875070Z","shell.execute_reply.started":"2024-11-05T15:43:44.868474Z","shell.execute_reply":"2024-11-05T15:43:44.873882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GCN Implementation","metadata":{}},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom sklearn.metrics import precision_score, recall_score, f1_score\n\n# Define device as CPU\ndevice = torch.device(\"cpu\")\n\n# Reconstruct tensors on CPU to ensure a fresh start\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32)\nX_val_tensor = torch.tensor(X_val, dtype=torch.float32)\nedge_index = torch.tensor(edge_index, dtype=torch.int64)\n\n# Ensure label tensors are in the correct format for BCEWithLogitsLoss\ny_train_tensor = torch.tensor(y_train, dtype=torch.float32)\ny_val_tensor = torch.tensor(y_val, dtype=torch.float32)\n\n# Check unique indices in edge_index\nprint(f\"Unique indices in edge_index: {torch.unique(edge_index)}\")\nprint(f\"Maximum index in edge_index: {edge_index.max()}\")\nprint(f\"Number of nodes in X_val_tensor: {X_val_tensor.shape[0]}\")\n\n# Filter out invalid edges\nmax_index = X_val_tensor.shape[0] - 1\nvalid_edges = edge_index[:, (edge_index[0] <= max_index) & (edge_index[1] <= max_index)]\n\n# Print filtered edge index information\nprint(f\"Filtered edge_index: shape = {valid_edges.shape}, unique indices = {torch.unique(valid_edges)}\")\n\n# Evaluation on CPU\nmodel.eval()\nwith torch.no_grad():\n    val_out = model(X_val_tensor, valid_edges)  # Use filtered edge index\n    val_out = torch.sigmoid(val_out)\n\n    # Convert predictions and labels to numpy arrays for metric calculation\n    val_out_np = val_out.numpy()\n    y_val_np = y_val_tensor.numpy()\n\n    # Apply binary threshold for classification\n    y_pred = (val_out_np >= 0.5).astype(int)\n    accuracy = (y_pred.flatten() == y_val_np).mean()\n    print(f\"Validation Accuracy on CPU: {accuracy:.4f}\")\n\n    # Additional metrics using sklearn\n    precision = precision_score(y_val_np, y_pred)\n    recall = recall_score(y_val_np, y_pred)\n    f1 = f1_score(y_val_np, y_pred)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:43:44.876330Z","iopub.execute_input":"2024-11-05T15:43:44.876703Z","iopub.status.idle":"2024-11-05T15:43:45.053855Z","shell.execute_reply.started":"2024-11-05T15:43:44.876663Z","shell.execute_reply":"2024-11-05T15:43:45.052597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Improving the Model:","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch_geometric.nn import GCNConv\nfrom torch_geometric.utils import to_undirected\nfrom sklearn.metrics import roc_auc_score, log_loss, accuracy_score, precision_score, recall_score, f1_score\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch_geometric.utils import add_self_loops\n\n# Define the enhanced GCN model with multi-layer and specified embedding\nclass EnhancedGCNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim, num_layers=3, dropout=0.5):\n        super(EnhancedGCNModel, self).__init__()\n        self.embedding_layer = nn.Linear(input_dim, 8)  # Embedding layer\n        self.convs = nn.ModuleList()\n        self.bns = nn.ModuleList()\n        \n        # Define multiple GCN layers as per the paper\n        for i in range(num_layers):\n            in_dim = 8 if i == 0 else hidden_dim\n            self.convs.append(GCNConv(in_dim, hidden_dim))\n            self.bns.append(nn.BatchNorm1d(hidden_dim))\n        \n        self.fc = nn.Linear(hidden_dim, output_dim)\n        self.dropout = nn.Dropout(dropout)\n        self.activation = nn.ReLU()\n\n    def forward(self, x, edge_index):\n        x = self.embedding_layer(x)\n        \n        for conv, bn in zip(self.convs, self.bns):\n            x = conv(x, edge_index)\n            x = bn(x)\n            x = self.activation(x)\n            x = self.dropout(x)\n        \n        x = self.fc(x)\n        return x\n\n# Training function with batch-wise edge filtering\ndef train_model(model, train_loader, edge_index, optimizer, criterion, num_epochs=100):\n    model.train()\n    for epoch in range(num_epochs):\n        total_loss = 0\n        for batch_x, batch_y in train_loader:\n            batch_nodes = batch_x.shape[0]\n            \n            # Filter edge_index for the batch to only contain within-batch edges\n            mask = (edge_index[0] < batch_nodes) & (edge_index[1] < batch_nodes)\n            batch_edge_index = edge_index[:, mask]\n            \n            optimizer.zero_grad()\n            out = model(batch_x, batch_edge_index)\n            loss = criterion(out.view(-1), batch_y.float())\n            loss.backward()\n            optimizer.step()\n            \n            total_loss += loss.item()\n        \n        # Print epoch loss every 10 epochs\n        if epoch % 10 == 0:\n            avg_loss = total_loss / len(train_loader)\n            print(f'Epoch [{epoch}/{num_epochs}], Loss: {avg_loss:.4f}')\n            \ndef evaluate_model(model, X_val_tensor, edge_index, y_val_tensor):\n    model.eval()\n    with torch.no_grad():\n        # Map nodes in X_val_tensor to the corresponding range in edge_index\n        unique_nodes = torch.arange(X_val_tensor.size(0), device=edge_index.device)\n        node_map = {n.item(): i for i, n in enumerate(unique_nodes)}\n\n        # Filter edge_index to only include edges within the validation set\n        mask = (edge_index[0] < X_val_tensor.size(0)) & (edge_index[1] < X_val_tensor.size(0))\n        filtered_edge_index = edge_index[:, mask]\n        \n        # Map the edges to the new subset of nodes\n        filtered_edge_index = torch.stack([\n            torch.tensor([node_map.get(n.item(), -1) for n in filtered_edge_index[0]]),\n            torch.tensor([node_map.get(n.item(), -1) for n in filtered_edge_index[1]])\n        ], dim=0)\n        \n        # Remove any edges with -1 (those outside the mapped range)\n        valid_edges = (filtered_edge_index != -1).all(dim=0)\n        filtered_edge_index = filtered_edge_index[:, valid_edges]\n        \n        # Forward pass with filtered edge_index\n        out = model(X_val_tensor, filtered_edge_index)\n        out_sigmoid = torch.sigmoid(out).view(-1).cpu().numpy()\n        y_true = y_val_tensor.cpu().numpy()\n\n        # Calculate evaluation metrics\n        auc = roc_auc_score(y_true, out_sigmoid)\n        logloss = log_loss(y_true, out_sigmoid)\n        accuracy = accuracy_score(y_true, (out_sigmoid >= 0.5).astype(int))\n       \n\n        print(f'Validation AUC: {auc:.4f}')\n        print(f'Validation Log Loss: {logloss:.4f}')\n        print(f'Validation Accuracy: {accuracy:.4f}')\n        \n        return auc, logloss, accuracy\n    # Prepare data for training and validation\n# Assume X_train_tensor, y_train_tensor, X_val_tensor, y_val_tensor, edge_index are defined elsewhere\n# Update edge_index to undirected and with self-loops\nedge_index = to_undirected(edge_index)\nedge_index, _ = add_self_loops(edge_index)\n\n# Define parameters\ninput_dim = X_train_tensor.shape[1]\nhidden_dim = 64\noutput_dim = 1\nbatch_size = 1024\nlearning_rate = 0.01\nnum_epochs = 100\n\n# Instantiate model, optimizer, and loss function\nmodel = EnhancedGCNModel(input_dim=input_dim, hidden_dim=hidden_dim, output_dim=output_dim)\noptimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\ncriterion = nn.BCEWithLogitsLoss()\n\n# Set up DataLoader for batch training\ntrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n\n# Train the model\ntrain_model(model, train_loader, edge_index, optimizer, criterion, num_epochs)\n\n# Evaluate the model on validation data\nevaluate_model(model, X_val_tensor, edge_index, y_val_tensor)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:43:45.055297Z","iopub.execute_input":"2024-11-05T15:43:45.055706Z","iopub.status.idle":"2024-11-05T15:45:40.842836Z","shell.execute_reply.started":"2024-11-05T15:43:45.055661Z","shell.execute_reply":"2024-11-05T15:45:40.841830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GNN model implementation","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch_geometric.nn import SAGEConv  # Change to other layers if needed\nfrom torch_geometric.utils import to_undirected, add_self_loops\nfrom sklearn.metrics import roc_auc_score, log_loss, accuracy_score\nfrom torch.utils.data import DataLoader, TensorDataset\n\n# Define the generic GNN model with multi-layer capability\nclass EnhancedGNNModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim, num_layers=3, dropout=0.5):\n        super(EnhancedGNNModel, self).__init__()\n        self.embedding_layer = nn.Linear(input_dim, 8)  # Embedding layer\n        self.convs = nn.ModuleList()\n        self.bns = nn.ModuleList()\n        \n        # Define multiple GNN layers\n        for i in range(num_layers):\n            in_dim = 8 if i == 0 else hidden_dim\n            self.convs.append(SAGEConv(in_dim, hidden_dim))\n            self.bns.append(nn.BatchNorm1d(hidden_dim))\n        \n        self.fc = nn.Linear(hidden_dim, output_dim)\n        self.dropout = nn.Dropout(dropout)\n        self.activation = nn.ReLU()\n\n    def forward(self, x, edge_index):\n        x = self.embedding_layer(x)\n        \n        for conv, bn in zip(self.convs, self.bns):\n            x = conv(x, edge_index)\n            x = bn(x)\n            x = self.activation(x)\n            x = self.dropout(x)\n        \n        x = self.fc(x)\n        return x\n\n# Training function with batch-wise edge filtering\ndef train_model(model, train_loader, edge_index, optimizer, criterion, num_epochs=100):\n    model.train()\n    for epoch in range(num_epochs):\n        total_loss = 0\n        for batch_x, batch_y in train_loader:\n            batch_nodes = batch_x.shape[0]\n            \n            # Filter edge_index for the batch to only contain within-batch edges\n            mask = (edge_index[0] < batch_nodes) & (edge_index[1] < batch_nodes)\n            batch_edge_index = edge_index[:, mask]\n            \n            optimizer.zero_grad()\n            out = model(batch_x, batch_edge_index)\n            loss = criterion(out.view(-1), batch_y.float())\n            loss.backward()\n            optimizer.step()\n            \n            total_loss += loss.item()\n        \n        # Print epoch loss every 10 epochs\n        if epoch % 10 == 0:\n            avg_loss = total_loss / len(train_loader)\n            print(f'Epoch [{epoch}/{num_epochs}], Loss: {avg_loss:.4f}')\n\ndef evaluate_model(model, X_val_tensor, edge_index, y_val_tensor):\n    model.eval()\n    with torch.no_grad():\n        # Map node indices in X_val_tensor to global indices\n        node_map = {node.item(): i for i, node in enumerate(torch.arange(X_val_tensor.shape[0]))}\n        \n        # Filter edge_index for the batch to include only nodes in X_val_tensor\n        mask = (edge_index[0] < X_val_tensor.shape[0]) & (edge_index[1] < X_val_tensor.shape[0])\n        filtered_edge_index = edge_index[:, mask]\n        \n        out = model(X_val_tensor, filtered_edge_index)\n        out_sigmoid = torch.sigmoid(out).view(-1).cpu().numpy()\n        y_true = y_val_tensor.cpu().numpy()\n\n        # Calculate only AUC, Log Loss, and Accuracy\n        auc = roc_auc_score(y_true, out_sigmoid)\n        logloss = log_loss(y_true, out_sigmoid)\n        accuracy = accuracy_score(y_true, (out_sigmoid >= 0.5).astype(int))\n        \n        print(f'Validation AUC: {auc:.4f}')\n        print(f'Validation Log Loss: {logloss:.4f}')\n        print(f'Validation Accuracy: {accuracy:.4f}')\n        \n        return auc, logloss, accuracy\n\n\n# Prepare data for training and validation\n# Assume X_train_tensor, y_train_tensor, X_val_tensor, y_val_tensor, edge_index are defined elsewhere\n# Update edge_index to undirected and with self-loops\nedge_index = to_undirected(edge_index)\nedge_index, _ = add_self_loops(edge_index)\n\n# Define parameters\ninput_dim = X_train_tensor.shape[1]\nhidden_dim = 64\noutput_dim = 1\nbatch_size = 1024\nlearning_rate = 0.01\nnum_epochs = 100\n\n# Instantiate model, optimizer, and loss function\nmodel = EnhancedGNNModel(input_dim=input_dim, hidden_dim=hidden_dim, output_dim=output_dim)\noptimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\ncriterion = nn.BCEWithLogitsLoss()\n\n# Set up DataLoader for batch training\ntrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n\n# Train the model\ntrain_model(model, train_loader, edge_index, optimizer, criterion, num_epochs)\n\n# Evaluate the model on validation data\nevaluate_model(model, X_val_tensor, edge_index, y_val_tensor)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T15:45:40.844546Z","iopub.execute_input":"2024-11-05T15:45:40.844887Z","iopub.status.idle":"2024-11-05T15:46:33.727240Z","shell.execute_reply.started":"2024-11-05T15:45:40.844852Z","shell.execute_reply":"2024-11-05T15:46:33.726209Z"},"trusted":true},"execution_count":null,"outputs":[]}]}