{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":113366,"databundleVersionId":13605326,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"09233fce","cell_type":"code","source":"import numpy as np\nimport math\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.optim as optim\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T17:34:33.986197Z","iopub.execute_input":"2025-09-19T17:34:33.986527Z","iopub.status.idle":"2025-09-19T17:34:41.266336Z","shell.execute_reply.started":"2025-09-19T17:34:33.986501Z","shell.execute_reply":"2025-09-19T17:34:41.264865Z"}},"outputs":[],"execution_count":null},{"id":"3498a743-66b4-414a-9c4c-45c110874fb7","cell_type":"markdown","source":"## Histogram showing the distribution of comment lengths.","metadata":{}},{"id":"80dd41d0","cell_type":"code","source":"try:\n    df = pd.read_csv(\"/kaggle/input/comments-classification/Dataset/train.csv\")\n    train_df, _ = train_test_split(df, test_size=0.2, random_state=42)\n\n    # Calculate the length of each comment (number of words)\n    train_df['comment_length'] = train_df['comment_text'].str.lower().str.strip().str.split().str.len()\n    \n    print(\"--- Descriptive Statistics for Comment Length ---\")\n\n    stats = train_df['comment_length'].describe(percentiles=[.50, .90, .95, .99])\n    print(stats)\n    print(\"-\" * 45)\n\n    print(\"\\nGenerating plot...\")\n    plt.style.use('seaborn-v0_8-whitegrid')\n    plt.figure(figsize=(14, 7))\n\n    # Plot the main histogram, focusing the range for better visibility\n    plt.hist(train_df['comment_length'], bins=100, edgecolor='black', alpha=0.7, range=(0, 400))\n\n    plt.title('Distribution of Comment Lengths in Training Data', fontsize=16)\n    plt.xlabel('Number of Words (Tokens)', fontsize=12)\n    plt.ylabel('Frequency (Number of Comments)', fontsize=12)\n    plt.legend()\n    plt.grid(True, which='both', linestyle='--', linewidth=0.5)\n\n    # Show the plot\n    plt.show()\n\nexcept FileNotFoundError:\n    print(\"Error: 'train.csv' not found. Please make sure the file is in the correct directory.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T17:35:31.151826Z","iopub.execute_input":"2025-09-19T17:35:31.152140Z","iopub.status.idle":"2025-09-19T17:35:36.127087Z","shell.execute_reply.started":"2025-09-19T17:35:31.152119Z","shell.execute_reply":"2025-09-19T17:35:36.126003Z"}},"outputs":[],"execution_count":null},{"id":"e4e1dad7-0be2-4d88-9c4d-6b0855a52c18","cell_type":"markdown","source":"# Vocabulary Creation\n* ### The text is cleaned by converting it to lowercase and stripping whitespace.\n\n* ### A vocabulary is built by tokenizing all training comments, counting word frequencies, and selecting the top 50,000 most common tokens.\n\n* ### Special tokens like [PAD], [UNK], and [CLS] are added to the vocabulary to handle padding, unknown words, and classification tasks, respectively.\n\n","metadata":{}},{"id":"d211c6cd","cell_type":"code","source":"import pandas as pd\nfrom collections import Counter\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n# Use sklearn for a clean initial split\nfrom sklearn.model_selection import train_test_split\n\ndf = pd.read_csv(\"t/kaggle/input/comments-classification/Dataset/train.csv\")\ntrain_df, val_df = train_test_split(df, test_size=0.2, random_state=42)\n\nprint(f\"Original df size: {len(df)}\")\nprint(f\"Training df size: {len(train_df)}\")\nprint(f\"Validation df size: {len(val_df)}\")\n\ntrain_df['comment_text'] = train_df['comment_text'].str.lower().str.strip()\ntokenized_comments = [comment.split() for comment in train_df['comment_text']]\nall_tokens = [t for comment in tokenized_comments for t in comment]\ntoken_counts = Counter(all_tokens)\n\nTOP_K = 50000\nmost_common_tokens = [tok for tok, _ in token_counts.most_common(TOP_K)]\nvocab = {tok: idx+2 for idx, tok in enumerate(most_common_tokens)}\nvocab['[PAD]'] = 0\nvocab['[UNK]'] = 1\nvocab['[CLS]'] = len(vocab)\n\nprint(\"Reduced vocab size:\", len(vocab))","metadata":{},"outputs":[],"execution_count":null},{"id":"621cfa12-7f72-48ed-88c1-01e5898b1b34","cell_type":"markdown","source":"### A custom CommentDataset class is defined. Its main job is to take a comment, tokenize it, convert the tokens to their corresponding IDs from the vocabulary, and pad or truncate the sequence to a fixed length.\n### To handle the class imbalance in the dataset, a WeightedRandomSampler is created. This sampler oversamples the minority class during training, ensuring the model sees a more balanced distribution of examples in each batch.","metadata":{}},{"id":"2c336c4a","cell_type":"code","source":"class CommentDataset(Dataset):\n    def __init__(self, dataframe, vocab, max_len):\n        self.comments = dataframe['comment_text'].tolist()\n        self.labels = dataframe['psychotic_depression'].tolist()\n        self.vocab = vocab\n        self.max_len = max_len\n    \n    def __len__(self):\n        return len(self.comments)\n    \n    def __getitem__(self, idx):\n        comment = self.comments[idx]\n        label = self.labels[idx]\n\n        # Tokenize, numericalize, and pad/truncate the single comment\n        tokens = comment.split()\n        token_ids = [self.vocab.get(t, self.vocab['[UNK]']) for t in tokens]\n        cls_token_id = self.vocab['[CLS]']\n        token_ids = [cls_token_id] + token_ids\n        \n        if len(token_ids) < self.max_len:\n            token_ids += [self.vocab['[PAD]']] * (self.max_len - len(token_ids))\n        else:\n            token_ids = token_ids[:self.max_len]\n        \n        return torch.tensor(token_ids), torch.tensor(label, dtype=torch.float32)","metadata":{},"outputs":[],"execution_count":null},{"id":"bce9910b","cell_type":"code","source":"d_model = 512\nh = 8               # number of attention heads\nvocab_size = len(vocab)  # reduced vocab\nmax_len = 100        # max tokens per comment\ndropout = 0.3\nbatch_size = 16 \nlr = 1e-5\nepochs = 20\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{},"outputs":[],"execution_count":null},{"id":"1f271192","cell_type":"code","source":"val_df['comment_text'] = val_df['comment_text'].str.lower().str.strip()\nfrom torch.utils.data import DataLoader, WeightedRandomSampler\n\n# compute class counts\nclass_counts = train_df[\"psychotic_depression\"].value_counts().to_dict()\nneg_count = class_counts[0]\npos_count = class_counts[1]\n\n# compute sample weights \nweights = [1/neg_count if label == 0 else 1/pos_count for label in train_df[\"psychotic_depression\"]]\n\n# create sampler\nsampler = WeightedRandomSampler(weights, num_samples=len(weights), replacement=True)\ntrain_dataset = CommentDataset(train_df, vocab, MAX_LEN)\nval_dataset = CommentDataset(val_df, vocab, MAX_LEN)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=sampler)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"1f1a7c61-8f21-47f3-aac9-ee290d68773a","cell_type":"markdown","source":"# Transformer Encoder Model Architecture","metadata":{}},{"id":"6196f07b","cell_type":"code","source":"def positional_encodings(seq_len, d_model):\n    pos = np.arange(seq_len)[:, np.newaxis]  #[[0], [1], [2], ..., [seq_len - 1]]\n    dims = np.arange(d_model)[np.newaxis, :] # [[0, 1, 2, ..., d_model - 1]]\n    \n    angles_input = pos/(np.power(10000, (2*(dims//2))/d_model))\n    pos_encodings = np.zeros((seq_len, d_model))\n    pos_encodings[:, 0::2] = np.sin(angles_input[:, 0::2])\n    pos_encodings[:, 1::2] = np.cos(angles_input[:, 1::2])\n\n    return torch.tensor(pos_encodings, dtype=torch.float32)","metadata":{},"outputs":[],"execution_count":null},{"id":"5b39d336","cell_type":"code","source":"class MultiHeadAttention(nn.Module):\n    def __init__(self, d_model, h):\n        super().__init__()\n        self.d_model = d_model\n        self.h = h\n        self.d_k = d_model // h \n        \n        #Linear projections for Q, K, V\n        self.Wq = nn.Linear(d_model, d_model)\n        self.Wk = nn.Linear(d_model, d_model)\n        self.Wv = nn.Linear(d_model, d_model)\n        \n        #Final linear projection\n        self.W_o = nn.Linear(d_model, d_model)\n        \n    def forward(self, x):\n        batch_size, seq_len, _ = x.size()\n        \n        #Project Q, K, V\n        Q = self.Wq(x)  \n        K = self.Wk(x)\n        V = self.Wv(x)\n        \n        # Split into h heads\n        Q = Q.view(batch_size, seq_len, self.h, self.d_k).transpose(1,2)  \n        K = K.view(batch_size, seq_len, self.h, self.d_k).transpose(1,2)\n        V = V.view(batch_size, seq_len, self.h, self.d_k).transpose(1,2)\n        \n        # Scaled dot-product attention\n        scores = torch.matmul(Q, K.transpose(-2,-1)) / math.sqrt(self.d_k)  \n        attn = torch.matmul(scores.softmax(dim=-1), V)\n        \n        # Concatenate heads\n        attn = attn.transpose(1,2).contiguous().view(batch_size, seq_len, self.d_model) \n        \n        # Final projection\n        out = self.W_o(attn)  # (batch, seq_len, d_model)\n        return out\n","metadata":{},"outputs":[],"execution_count":null},{"id":"648840d4","cell_type":"code","source":"class FeedFwd(nn.Module):\n    def __init__(self, d_model):\n        super().__init__()\n        self.linear1 = nn.Linear(d_model, d_model*2)\n        self.linear2 = nn.Linear(2*d_model, d_model)\n    \n    def forward(self, x):\n        x = nn.functional.relu(self.linear1(x))\n        x = self.linear2(x)\n        return x","metadata":{},"outputs":[],"execution_count":null},{"id":"f0e977a2","cell_type":"code","source":"class EncoderBlock(nn.Module):\n    def __init__(self, d_model, h):\n        super().__init__()\n        self.mha = MultiHeadAttention(d_model, h)\n        self.norm1 = nn.LayerNorm(d_model)\n        self.ffd = FeedFwd(d_model)       \n        self.norm2 = nn.LayerNorm(d_model)\n\n    def forward(self, x):\n        x = self.norm1(x + self.mha(x))  \n        x = self.norm2(x + self.ffd(x))  \n\n        return x","metadata":{},"outputs":[],"execution_count":null},{"id":"7a82eeb6-41cd-4c97-bb61-d3d42a9dcc49","cell_type":"markdown","source":"## The final architecture consists of an embedding layer, the addition of positional encodings, and three stacked EncoderBlocks. The output corresponding to the special [CLS] token is then fed into a final linear classifier to produce a single logit for the binary classification task.","metadata":{}},{"id":"ac9d8460","cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, vocab_size, d_model, h, max_len=512, dropout=0.2):\n        super().__init__()\n        self.d_model = d_model\n\n        self.embeddings = nn.Embedding(vocab_size, d_model)\n\n        self.block1 = EncoderBlock(d_model, h)\n        self.block2 = EncoderBlock(d_model, h)\n        self.block3 = EncoderBlock(d_model, h)\n\n        self.ffd_block = nn.Sequential(\n            nn.Linear(d_model, 2*d_model),\n            nn.LeakyReLU(),\n            nn.Dropout(dropout),\n\n            nn.Linear(2*d_model, d_model),\n            nn.LeakyReLU(),\n            nn.Dropout(dropout),\n\n            nn.Linear(d_model, 1)   \n        )\n\n    def forward(self, x):\n        # x: (batch, seq_len)\n        x = self.embeddings(x) * (self.d_model ** 0.5)\n        pos_enc = positional_encodings(x.size(1), self.d_model).to(x.device)  # (seq_len, d_model)\n        x = x + pos_enc.unsqueeze(0)  # broadcast over batch\n\n\n        # pass through encoder blocks\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n\n        x = x[:,0,:]   \n\n        x = self.ffd_block(x)\n        return x\n","metadata":{},"outputs":[],"execution_count":null},{"id":"9815452d","cell_type":"code","source":"model = Model(vocab_size, d_model, h, max_len, dropout).to(device)\nneg_samples = df['psychotic_depression'].value_counts()[0]\npos_samples = df['psychotic_depression'].value_counts()[1]\nweight = torch.tensor([neg_samples / pos_samples], device=device)\ncriterion = nn.BCEWithLogitsLoss(pos_weight=weight)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"3de880ca","cell_type":"code","source":"optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=5e-3)","metadata":{},"outputs":[],"execution_count":null},{"id":"3bcd4201-36cb-4ea2-a5de-6bc76a8b8776","cell_type":"code","source":"for epoch in range(epochs):\n    model.train()\n    total_train_loss = 0\n    \n    train_progress_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs} [Train]\", leave=False)\n\n    for batch_tokens, batch_labels in train_progress_bar:\n        batch_tokens = batch_tokens.to(device)\n        batch_labels = batch_labels.to(device)\n\n        optimizer.zero_grad()\n        outputs = model(batch_tokens).squeeze(1)\n        loss = criterion(outputs, batch_labels)\n        \n        loss.backward()\n        optimizer.step()\n\n        total_train_loss += loss.item() * batch_tokens.size(0)\n        train_progress_bar.set_postfix(loss=f\"{loss.item():.4f}\")\n\n    avg_train_loss = total_train_loss / len(train_dataset)\n    \n    model.eval()\n    total_val_loss = 0\n    all_labels = []\n    all_preds = []\n    \n    val_progress_bar = tqdm(val_loader, desc=f\"Epoch {epoch+1}/{epochs} [Val]\", leave=False)\n\n    with torch.no_grad():\n        for batch_tokens, batch_labels in val_progress_bar:\n            batch_tokens = batch_tokens.to(device)\n            batch_labels = batch_labels.to(device)\n\n            outputs = model(batch_tokens).squeeze(1)\n            loss = criterion(outputs, batch_labels)\n            \n            total_val_loss += loss.item() * batch_tokens.size(0)\n            probs = torch.sigmoid(outputs)\n            preds = (probs > 0.4).int()\n            \n            all_preds.extend(preds.cpu().numpy())\n            all_labels.extend(batch_labels.cpu().numpy())\n\n            val_progress_bar.set_postfix(loss=f\"{loss.item():.4f}\")\n            \n    avg_val_loss = total_val_loss / len(val_dataset)\n    \n    # Calculate F1 score from all collected predictions and labels\n    val_f1 = f1_score(all_labels, all_preds)\n    \n    # Print summary for the epoch\n    print(f\"Epoch {epoch+1}/{epochs} | Avg Train Loss: {avg_train_loss:.4f} | Avg Val Loss: {avg_val_loss:.4f} | Val F1: {val_f1:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"1ed8c9b5","cell_type":"code","source":"torch.save({\n            'model_state_dict': model.state_dict(),\n            }, 'model_params.pth')\n    ","metadata":{},"outputs":[],"execution_count":null},{"id":"429abf02","cell_type":"code","source":"model = Model(vocab_size, d_model, h, max_len, dropout).to(device)\ncheckpoint = torch.load('model_params.pth')\nmodel.load_state_dict(checkpoint['model_state_dict'])","metadata":{},"outputs":[],"execution_count":null},{"id":"a14b4665","cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/comments-classification/Dataset/test.csv\")\ntest_df['comment_text'] = test_df['comment_text'].str.lower().str.strip()\ntest_df['psychotic_depression'] = 0 \ntest_dataset = CommentDataset(test_df, vocab, MAX_LEN)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\nmodel.eval() # IMPORTANT: Set the model to evaluation mode\n\n# 2. Run the inference loop\nall_predictions = []\nwith torch.no_grad(): # Disable gradient calculations\n    for batch_tokens, _ in tqdm(test_loader, desc=\"Generating Predictions\"):\n        batch_tokens = batch_tokens.to(device)\n        \n        # Get model outputs (logits)\n        outputs = model(batch_tokens).squeeze(1)\n        \n        # Convert logits to probabilities and then to binary predictions\n        probs = torch.sigmoid(outputs)\n        preds = (probs > 0.4).int() # Using a 0.5 threshold\n        \n        # Store the predictions\n        all_predictions.extend(preds.cpu().numpy())\n\n# Add the predictions as a new column\ntest_df['psychotic_depression'] = all_predictions\n\n# Save the final results to a CSV file\n# You might only need the ID and the prediction columns for submission\nsubmission_df = test_df[['id', 'psychotic_depression']] # Adjust columns as needed\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"Submission file created successfully!\")\nprint(submission_df.head())","metadata":{},"outputs":[],"execution_count":null},{"id":"c6ea6dae","cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}