{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96745,"databundleVersionId":11513148,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport torch\nimport torch.nn as nn\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom transformers import GPT2Tokenizer\nfrom sklearn.model_selection import train_test_split\nimport pickle\nimport numpy as np\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv_path = \"/kaggle/input/automated-video-captioning/train.csv\"\ntest_csv_path = \"/kaggle/input/automated-video-captioning/test.csv\"\ntrain_dir = \"/kaggle/input/automated-video-captioning/train_videos\"\ntest_dir = \"/kaggle/input/automated-video-captioning/test_videos\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Extractor (for pic embeddings)","metadata":{}},{"cell_type":"code","source":"def get_video_resnet_features(\n    path_to_video,\n    interval=25,\n    device=\"cuda\"\n):\n    try:\n        base_model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1).to(device)\n        feature_extractor = nn.Sequential(*list(base_model.children())[:-1])\n        feature_extractor.eval()\n\n        transform = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize((224, 224)),\n            transforms.ToTensor(),\n            transforms.Normalize([0.485, 0.456, 0.406],\n                                 [0.229, 0.224, 0.225])\n        ])\n\n        vid = cv2.VideoCapture(path_to_video)\n        if not vid.isOpened():\n            raise IOError(f\"Failed to load video: {path_to_video}\")\n\n        features = []\n        idx = 0\n\n        while True:\n            success, frame = vid.read()\n            if not success:\n                break\n            if idx % interval == 0:\n                frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                input_tensor = transform(frame_rgb).unsqueeze(0).to(device)\n                with torch.no_grad():\n                    vec = feature_extractor(input_tensor).squeeze()\n                    features.append(vec.cpu())\n            idx += 1\n\n        vid.release()\n\n        if not features:\n            raise RuntimeError(f\"No valid frames were processed in: {path_to_video}\")\n\n        return torch.stack(features)\n\n    except Exception as error:\n        print(f\"Failed to process {path_to_video}: {error}\")\n        return None\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Bleu Loss","metadata":{}},{"cell_type":"code","source":"from nltk.translate.bleu_score import corpus_bleu\n\ndef compute_bleu_score(pred_texts, true_texts):\n    references = [[t.split()] for t in true_texts]\n    hypotheses = [p.split() for p in pred_texts]\n    return corpus_bleu(references, hypotheses, weights=(0.25, 0.25, 0.25, 0.25))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# My Model","metadata":{}},{"cell_type":"code","source":"class GRUModel(nn.Module):\n    def __init__(\n        self,\n        embedding_dim=2048,\n        hidden_dim=256,\n        num_layers=2,\n        vocab_size=50257,\n        max_len=50,\n        dropout=0.1,\n    ):\n        super(GRUModel, self).__init__()\n        self.embedding_dim = embedding_dim\n        self.hidden_dim = hidden_dim\n        self.max_len = max_len\n\n        self.video_gru = nn.GRU(\n            input_size=embedding_dim,\n            hidden_size=hidden_dim,\n            num_layers=num_layers,\n            batch_first=True,\n            dropout=dropout if num_layers > 1 else 0,\n        )\n\n        self.text_embedding = nn.Embedding(vocab_size, hidden_dim)\n        self.text_gru = nn.GRU(\n            input_size=hidden_dim,\n            hidden_size=hidden_dim,\n            num_layers=num_layers,\n            batch_first=True,\n            dropout=dropout if num_layers > 1 else 0,\n        )\n        self.fc_out = nn.Linear(hidden_dim, vocab_size)\n\n    def forward(self, video_emb, text_ids):\n        batch_size = video_emb.size(0)\n\n        _, video_hidden = self.video_gru(video_emb)\n\n        text_emb = self.text_embedding(text_ids)\n        output, _ = self.text_gru(text_emb, video_hidden)\n        return self.fc_out(output)\n\n    def generate(\n        self,\n        video_emb,\n        tokenizer,\n        max_len=50,\n        device=\"cuda\",\n    ):\n        batch_size = video_emb.size(0)\n        _, video_hidden = self.video_gru(video_emb)\n\n        generated = torch.full(\n            (batch_size, 1), tokenizer.bos_token_id, dtype=torch.long, device=device\n        )\n        for _ in range(max_len):\n            text_emb = self.text_embedding(generated)\n            output, video_hidden = self.text_gru(text_emb[:, -1:, :], video_hidden)\n            logits = self.fc_out(output[:, -1, :])\n            next_token = torch.argmax(logits, dim=-1).unsqueeze(1)\n            generated = torch.cat([generated, next_token], dim=1)\n            if (next_token == tokenizer.eos_token_id).any():\n                break\n        return generated","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    embeddings, text_ids = zip(*batch)\n    padded_embeddings = nn.utils.rnn.pad_sequence(embeddings, batch_first=True)\n    padded_text_ids = nn.utils.rnn.pad_sequence(\n        text_ids, batch_first=True, padding_value=0\n    )\n    return padded_embeddings, padded_text_ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T12:24:00.818482Z","iopub.execute_input":"2025-06-23T12:24:00.818760Z","iopub.status.idle":"2025-06-23T12:24:00.822910Z","shell.execute_reply.started":"2025-06-23T12:24:00.818739Z","shell.execute_reply":"2025-06-23T12:24:00.822154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class VideoDataset(Dataset):\n    def __init__(\n        self,\n        video_paths,\n        captions,\n        tokenizer,\n        device=\"cuda\",\n        cache_dir=\"cache\",\n        max_len=50,\n    ):\n        self.video_paths = video_paths\n        self.captions = captions\n        self.tokenizer = tokenizer\n        self.device = device\n        self.cache_dir = cache_dir\n        self.max_len = max_len\n        os.makedirs(cache_dir, exist_ok=True)\n        self.encoded_videos = self.load_embeddings()\n\n    def load_embeddings(self):\n        encoded_videos = {}\n        for video_path in tqdm(self.video_paths, desc=\"Loading embeddings\"):\n            cache_path = os.path.join(\n                self.cache_dir, f\"{os.path.basename(video_path)}.pkl\"\n            )\n            if os.path.exists(cache_path):\n                with open(cache_path, \"rb\") as f:\n                    encoded_videos[video_path] = pickle.load(f)\n            else:\n                embeddings = get_video_resnet_features(\n                    video_path, interval=24, device=self.device\n                )\n                if embeddings is not None:\n                    encoded_videos[video_path] = embeddings\n                    with open(cache_path, \"wb\") as f:\n                        pickle.dump(embeddings, f)\n        return encoded_videos\n\n    def __len__(self):\n        return len(self.video_paths)\n\n    def __getitem__(self, idx):\n        video_path = self.video_paths[idx]\n        embeddings = self.encoded_videos.get(video_path)\n        if embeddings is None:\n            embeddings = torch.zeros((1, 2048))\n        caption = self.captions[idx]\n        tokens = self.tokenizer(\n            caption,\n            max_length=self.max_len,\n            padding=\"max_length\",\n            truncation=True,\n            return_tensors=\"pt\",\n        )\n        text_ids = tokens[\"input_ids\"].squeeze(0)\n        return embeddings, text_ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T12:24:02.704002Z","iopub.execute_input":"2025-06-23T12:24:02.704805Z","iopub.status.idle":"2025-06-23T12:24:02.712097Z","shell.execute_reply.started":"2025-06-23T12:24:02.704772Z","shell.execute_reply":"2025-06-23T12:24:02.711315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\ntokenizer.pad_token = tokenizer.eos_token\ntokenizer.bos_token_id = tokenizer.eos_token_id\n\ntrain_df = pd.read_csv(train_csv_path)\nvideo_paths = [os.path.join(train_dir, name) for name in train_df[\"file_name\"]]\ncaptions = train_df[\"caption\"].tolist()\n\ntrain_idx, val_idx = train_test_split(\n    range(len(video_paths)), test_size=0.2, random_state=42\n)\ntrain_paths = [video_paths[i] for i in train_idx]\ntrain_captions = [captions[i] for i in train_idx]\nval_paths = [video_paths[i] for i in val_idx]\nval_captions = [captions[i] for i in val_idx]\n\ntrain_dataset = VideoDataset(\n    train_paths, train_captions, tokenizer, device=device, cache_dir=\"embeddings_train\"\n)\nval_dataset = VideoDataset(\n    val_paths, val_captions, tokenizer, device=device, cache_dir=\"embeddings_val\"\n)\ntrain_loader = DataLoader(\n    train_dataset, batch_size=4, shuffle=True, collate_fn=collate_fn\n)\nval_loader = DataLoader(val_dataset, batch_size=4, shuffle=False, collate_fn=collate_fn)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T12:24:05.481568Z","iopub.execute_input":"2025-06-23T12:24:05.481840Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"model = GRUModel(\n    vocab_size=tokenizer.vocab_size, hidden_dim=256, dropout=0.2, num_layers=2\n).to(device)\n\ncriterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id)\noptimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=2e-4)\nnum_epochs = 50","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in tqdm(range(num_epochs), desc=\"Epochs\"):\n    model.train()\n    train_loss = 0\n    train_total = 0\n\n    for embeddings, text_ids in train_loader:\n        embeddings, text_ids = embeddings.to(device), text_ids.to(device)\n        optimizer.zero_grad()\n        outputs = model(embeddings, text_ids[:, :-1])\n        loss = criterion(\n            outputs.view(-1, tokenizer.vocab_size), text_ids[:, 1:].reshape(-1)\n        )\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item() * embeddings.size(0)\n        train_total += embeddings.size(0)\n\n    train_loss /= train_total\n\n    model.eval()\n    val_loss = 0\n    val_total = 0\n    pred_captions = []\n    true_captions = val_captions\n    with torch.no_grad():\n        for embeddings, text_ids in val_loader:\n            embeddings, text_ids = embeddings.to(device), text_ids.to(device)\n            outputs = model(embeddings, text_ids[:, :-1])\n            loss = criterion(\n                outputs.view(-1, tokenizer.vocab_size), text_ids[:, 1:].reshape(-1)\n            )\n            val_loss += loss.item() * embeddings.size(0)\n            val_total += embeddings.size(0)\n\n            generated_ids = model.generate(\n                embeddings, tokenizer, max_len=50, device=device\n            )\n            for ids in generated_ids:\n                caption = tokenizer.decode(ids, skip_special_tokens=True)\n                pred_captions.append(caption)\n\n    val_loss /= val_total\n    bleu_score = compute_bleu_score(pred_captions, true_captions)\n\n    print(f\"Epoch {epoch+1}/{num_epochs}:\"\n          f\" | Train Loss: {train_loss:.4f}\"\n          f\" | Val Loss: {val_loss:.4f}, BLEU Score: {bleu_score:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test Inference","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(test_csv_path)\ntest_paths = [os.path.join(test_dir, name) for name in test_df[\"file_name\"]]\ntest_dataset = VideoDataset(\n    test_paths,\n    [\"\"] * len(test_paths),\n    tokenizer,\n    device=device,\n    cache_dir=\"embeddings_test\",\n)\ntest_loader = DataLoader(\n    test_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\ncaps = []\nwith torch.no_grad():\n    for embeddings, _ in test_loader:\n        generated_ids = model.generate(embeddings.to(device), \n                                       tokenizer, \n                                       max_len=50, \n                                       device=device)\n        for ids in generated_ids:\n            caption = tokenizer.decode(ids, skip_special_tokens=True)\n            caps.append(caption)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ans = pd.DataFrame(\n    {\n        \"index\": test_df.index,\n        \"file_name\": test_df[\"file_name\"],\n        \"caption\": caps,\n    }\n)\nans.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}