{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96745,"databundleVersionId":11513148,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Libs","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport torch\nimport torch.nn as nn\nimport pandas as pd\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom transformers import GPT2Tokenizer\nfrom sklearn.model_selection import train_test_split\nimport pickle\nfrom nltk.translate.bleu_score import sentence_bleu\nimport numpy as np","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:57:52.198598Z","iopub.execute_input":"2025-05-13T19:57:52.199316Z","iopub.status.idle":"2025-05-13T19:58:04.491314Z","shell.execute_reply.started":"2025-05-13T19:57:52.199293Z","shell.execute_reply":"2025-05-13T19:58:04.490491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ntrain_csv_path = \"/kaggle/input/automated-video-captioning/train.csv\"\ntest_csv_path = \"/kaggle/input/automated-video-captioning/test.csv\"\ntrain_dir = \"/kaggle/input/automated-video-captioning/train_videos\"\ntest_dir = \"/kaggle/input/automated-video-captioning/test_videos\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:58:04.492544Z","iopub.execute_input":"2025-05-13T19:58:04.492962Z","iopub.status.idle":"2025-05-13T19:58:04.558694Z","shell.execute_reply.started":"2025-05-13T19:58:04.492940Z","shell.execute_reply":"2025-05-13T19:58:04.557887Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Picture embedding extractor","metadata":{}},{"cell_type":"code","source":"def extract_resnet_embeddings(\n    video_path,\n    frame_interval=25,\n    device=\"cuda\",\n):\n    try:\n        model = models.resnet50(pretrained=True).to(device)\n        model = nn.Sequential(*list(model.children())[:-1])\n        model.eval()\n        preprocess = transforms.Compose(\n            [\n                transforms.ToPILImage(),\n                transforms.Resize((224, 224)),\n                transforms.ToTensor(),\n                transforms.Normalize(\n                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]\n                ),\n            ]\n        )\n        video = cv2.VideoCapture(video_path)\n        if not video.isOpened():\n            raise ValueError(f\"Cannot open video: {video_path}\")\n        embeddings = []\n\n        frame_idx = 0\n        while True:\n            ret, frame = video.read()\n            if not ret:\n                break\n            if frame_idx % frame_interval == 0:\n                image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                image = preprocess(image).unsqueeze(0).to(device)\n                with torch.no_grad():\n                    embedding = model(image)\n                    embeddings.append(embedding.cpu().view(-1))\n            frame_idx += 1\n\n        video.release()\n        if not embeddings:\n            raise ValueError(f\"No frames extracted from {video_path}\")\n        return torch.stack(embeddings)\n    except Exception as e:\n        print(f\"Error processing {video_path}: {e}\")\n        return None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:58:04.559529Z","iopub.execute_input":"2025-05-13T19:58:04.559812Z","iopub.status.idle":"2025-05-13T19:58:04.582731Z","shell.execute_reply.started":"2025-05-13T19:58:04.559786Z","shell.execute_reply":"2025-05-13T19:58:04.582199Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model architecture\n\nPS: <br>\nvideo_hidden: (num_layers, batch_size, hidden_dim) <br>\ntext_emb: (batch_size, text_len, hidden_dim) <br>\noutput: (batch_size, text_len, hidden_dim) <br>\nself.fc_out(output): (batch_size, text_len, vocab_size) <br>\nvideo_hidden: (num_layers, batch_size, hidden_dim) <br>\ntext_emb: (batch_size, current_len, hidden_dim)<br>\noutput: (batch_size, 1, hidden_dim) <br>\nlogits: (batch_size, vocab_size) <br>\nnext_token: (batch_size, 1)","metadata":{}},{"cell_type":"code","source":"class ResNetGRUCaptioning(nn.Module):\n    def __init__(\n        self,\n        embedding_dim=2048,\n        hidden_dim=256,\n        num_layers=2,\n        vocab_size=50257,\n        max_len=50,\n        dropout=0.1,\n    ):\n        super(ResNetGRUCaptioning, self).__init__()\n        self.embedding_dim = embedding_dim\n        self.hidden_dim = hidden_dim\n        self.max_len = max_len\n\n        self.video_gru = nn.GRU(\n            input_size=embedding_dim,\n            hidden_size=hidden_dim,\n            num_layers=num_layers,\n            batch_first=True,\n            dropout=dropout if num_layers > 1 else 0,\n        )\n\n        self.text_embedding = nn.Embedding(vocab_size, hidden_dim)\n        self.text_gru = nn.GRU(\n            input_size=hidden_dim,\n            hidden_size=hidden_dim,\n            num_layers=num_layers,\n            batch_first=True,\n            dropout=dropout if num_layers > 1 else 0,\n        )\n        self.fc_out = nn.Linear(hidden_dim, vocab_size)\n\n    def forward(self, video_emb, text_ids):\n        batch_size = video_emb.size(0)\n\n        _, video_hidden = self.video_gru(video_emb)\n\n        text_emb = self.text_embedding(text_ids)\n        output, _ = self.text_gru(text_emb, video_hidden)\n        return self.fc_out(output)\n\n    def generate(\n        self,\n        video_emb,\n        tokenizer,\n        max_len=50,\n        device=\"cuda\",\n    ):\n        batch_size = video_emb.size(0)\n        _, video_hidden = self.video_gru(video_emb)\n\n        generated = torch.full(\n            (batch_size, 1), tokenizer.bos_token_id, dtype=torch.long, device=device\n        )\n        for _ in range(max_len):\n            text_emb = self.text_embedding(generated)\n            output, video_hidden = self.text_gru(text_emb[:, -1:, :], video_hidden)\n            logits = self.fc_out(output[:, -1, :])\n            next_token = torch.argmax(logits, dim=-1).unsqueeze(1)\n            generated = torch.cat([generated, next_token], dim=1)\n            if (next_token == tokenizer.eos_token_id).any():\n                break\n        return generated","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:58:04.584673Z","iopub.execute_input":"2025-05-13T19:58:04.584979Z","iopub.status.idle":"2025-05-13T19:58:04.601258Z","shell.execute_reply.started":"2025-05-13T19:58:04.584961Z","shell.execute_reply":"2025-05-13T19:58:04.600725Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Collator","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    embeddings, text_ids = zip(*batch)\n    padded_embeddings = nn.utils.rnn.pad_sequence(embeddings, batch_first=True)\n    padded_text_ids = nn.utils.rnn.pad_sequence(\n        text_ids, batch_first=True, padding_value=0\n    )\n    return padded_embeddings, padded_text_ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:58:04.601889Z","iopub.execute_input":"2025-05-13T19:58:04.602128Z","iopub.status.idle":"2025-05-13T19:58:04.622085Z","shell.execute_reply.started":"2025-05-13T19:58:04.602110Z","shell.execute_reply":"2025-05-13T19:58:04.621357Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Video storage and preprocessing (DataSet and DataLoader)","metadata":{}},{"cell_type":"code","source":"class VideoCaptionDataset(Dataset):\n    def __init__(\n        self,\n        video_paths,\n        captions,\n        tokenizer,\n        device=\"cuda\",\n        cache_dir=\"embeddings\",\n        max_len=50,\n    ):\n        self.video_paths = video_paths\n        self.captions = captions\n        self.tokenizer = tokenizer\n        self.device = device\n        self.cache_dir = cache_dir\n        self.max_len = max_len\n        os.makedirs(cache_dir, exist_ok=True)\n        self.encoded_videos = self.load_embeddings()\n\n    def load_embeddings(self):\n        encoded_videos = {}\n        for video_path in tqdm(self.video_paths, desc=\"Loading embeddings\"):\n            cache_path = os.path.join(\n                self.cache_dir, f\"{os.path.basename(video_path)}.pkl\"\n            )\n            if os.path.exists(cache_path):\n                with open(cache_path, \"rb\") as f:\n                    encoded_videos[video_path] = pickle.load(f)\n            else:\n                embeddings = extract_resnet_embeddings(\n                    video_path, frame_interval=25, device=self.device\n                )\n                if embeddings is not None:\n                    encoded_videos[video_path] = embeddings\n                    with open(cache_path, \"wb\") as f:\n                        pickle.dump(embeddings, f)\n        return encoded_videos\n\n    def __len__(self):\n        return len(self.video_paths)\n\n    def __getitem__(self, idx):\n        video_path = self.video_paths[idx]\n        embeddings = self.encoded_videos.get(video_path)\n        if embeddings is None:\n            embeddings = torch.zeros((1, 2048))\n        caption = self.captions[idx]\n        tokens = self.tokenizer(\n            caption,\n            max_length=self.max_len,\n            padding=\"max_length\",\n            truncation=True,\n            return_tensors=\"pt\",\n        )\n        text_ids = tokens[\"input_ids\"].squeeze(0)\n        return embeddings, text_ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:58:04.622739Z","iopub.execute_input":"2025-05-13T19:58:04.622941Z","iopub.status.idle":"2025-05-13T19:58:04.642179Z","shell.execute_reply.started":"2025-05-13T19:58:04.622925Z","shell.execute_reply":"2025-05-13T19:58:04.641443Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Some internet code of bleu score","metadata":{}},{"cell_type":"code","source":"def compute_bleu_score(pred_texts, true_texts):\n    scores = []\n    for pred, true in zip(pred_texts, true_texts):\n        pred_tokens = pred.split()\n        true_tokens = true.split()\n        score = sentence_bleu(\n            [true_tokens], pred_tokens, weights=(0.25, 0.25, 0.25, 0.25)\n        )\n        scores.append(score)\n    return np.mean(scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T19:58:04.643094Z","iopub.execute_input":"2025-05-13T19:58:04.643554Z","iopub.status.idle":"2025-05-13T19:58:04.662701Z","shell.execute_reply.started":"2025-05-13T19:58:04.643531Z","shell.execute_reply":"2025-05-13T19:58:04.661982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data processing\n\n1. Load and parse train.csv\n2. Split into train and validation\n3. Create datasets and dataloaders","metadata":{}},{"cell_type":"code","source":"tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\ntokenizer.pad_token = tokenizer.eos_token\ntokenizer.bos_token_id = tokenizer.eos_token_id\n\ntrain_df = pd.read_csv(train_csv_path)\nvideo_paths = [os.path.join(train_dir, name) for name in train_df[\"file_name\"]]\ncaptions = train_df[\"caption\"].tolist()\n\ntrain_idx, val_idx = train_test_split(\n    range(len(video_paths)), test_size=0.2, random_state=42\n)\ntrain_paths = [video_paths[i] for i in train_idx]\ntrain_captions = [captions[i] for i in train_idx]\nval_paths = [video_paths[i] for i in val_idx]\nval_captions = [captions[i] for i in val_idx]\n\ntrain_dataset = VideoCaptionDataset(\n    train_paths, train_captions, tokenizer, device=device, cache_dir=\"embeddings_train\"\n)\nval_dataset = VideoCaptionDataset(\n    val_paths, val_captions, tokenizer, device=device, cache_dir=\"embeddings_val\"\n)\ntrain_loader = DataLoader(\n    train_dataset, batch_size=4, shuffle=True, collate_fn=collate_fn\n)\nval_loader = DataLoader(val_dataset, batch_size=4, shuffle=False, collate_fn=collate_fn)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T20:02:01.905205Z","iopub.execute_input":"2025-05-13T20:02:01.905489Z","iopub.status.idle":"2025-05-13T20:07:41.199096Z","shell.execute_reply.started":"2025-05-13T20:02:01.905468Z","shell.execute_reply":"2025-05-13T20:07:41.198468Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *desided to change something at data processing","metadata":{}},{"cell_type":"code","source":"# !rm -rf /kaggle/working/embeddings_train\n# !rm -rf /kaggle/working/embeddings_val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T20:00:45.229966Z","iopub.status.idle":"2025-05-13T20:00:45.230324Z","shell.execute_reply.started":"2025-05-13T20:00:45.230126Z","shell.execute_reply":"2025-05-13T20:00:45.230159Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model training and validating","metadata":{}},{"cell_type":"code","source":"model = ResNetGRUCaptioning(\n    vocab_size=tokenizer.vocab_size, hidden_dim=256, dropout=0.2, num_layers=2\n).to(device)\n\ncriterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id)\noptimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=2e-4)\nnum_epochs = 50\n\nfor epoch in tqdm(range(num_epochs), desc=\"Epochs\"):\n    model.train()\n    train_loss = 0\n    train_total = 0\n\n    for embeddings, text_ids in train_loader:\n        embeddings, text_ids = embeddings.to(device), text_ids.to(device)\n        optimizer.zero_grad()\n        outputs = model(embeddings, text_ids[:, :-1])\n        loss = criterion(\n            outputs.view(-1, tokenizer.vocab_size), text_ids[:, 1:].reshape(-1)\n        )\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item() * embeddings.size(0)\n        train_total += embeddings.size(0)\n\n    train_loss /= train_total\n\n    model.eval()\n    val_loss = 0\n    val_total = 0\n    pred_captions = []\n    true_captions = val_captions\n    with torch.no_grad():\n        for embeddings, text_ids in val_loader:\n            embeddings, text_ids = embeddings.to(device), text_ids.to(device)\n            outputs = model(embeddings, text_ids[:, :-1])\n            loss = criterion(\n                outputs.view(-1, tokenizer.vocab_size), text_ids[:, 1:].reshape(-1)\n            )\n            val_loss += loss.item() * embeddings.size(0)\n            val_total += embeddings.size(0)\n\n            generated_ids = model.generate(\n                embeddings, tokenizer, max_len=50, device=device\n            )\n            for ids in generated_ids:\n                caption = tokenizer.decode(ids, skip_special_tokens=True)\n                pred_captions.append(caption)\n\n    val_loss /= val_total\n    bleu_score = compute_bleu_score(pred_captions, true_captions)\n\n    print(f\"Epoch {epoch+1}/{num_epochs}:\")\n    print(f\"Train Loss: {train_loss:.4f}\")\n    print(f\"Val Loss: {val_loss:.4f}, BLEU Score: {bleu_score:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T20:24:24.082030Z","iopub.execute_input":"2025-05-13T20:24:24.082767Z","iopub.status.idle":"2025-05-13T20:27:03.344132Z","shell.execute_reply.started":"2025-05-13T20:24:24.082741Z","shell.execute_reply":"2025-05-13T20:27:03.343415Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test DataSet init","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(test_csv_path)\ntest_paths = [os.path.join(test_dir, name) for name in test_df[\"file_name\"]]\ntest_dataset = VideoCaptionDataset(\n    test_paths,\n    [\"\"] * len(test_paths),\n    tokenizer,\n    device=device,\n    cache_dir=\"embeddings_test\",\n)\ntest_loader = DataLoader(\n    test_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T20:27:59.301769Z","iopub.execute_input":"2025-05-13T20:27:59.302068Z","iopub.status.idle":"2025-05-13T20:35:14.981126Z","shell.execute_reply.started":"2025-05-13T20:27:59.302044Z","shell.execute_reply":"2025-05-13T20:35:14.980551Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test prediction","metadata":{}},{"cell_type":"code","source":"model.eval()\npred_captions = []\nwith torch.no_grad():\n    for embeddings, _ in test_loader:\n        embeddings = embeddings.to(device)\n        generated_ids = model.generate(embeddings, tokenizer, max_len=50, device=device)\n        for ids in generated_ids:\n            caption = tokenizer.decode(ids, skip_special_tokens=True)\n            pred_captions.append(caption)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T20:35:14.982227Z","iopub.execute_input":"2025-05-13T20:35:14.982455Z","iopub.status.idle":"2025-05-13T20:35:17.408586Z","shell.execute_reply.started":"2025-05-13T20:35:14.982438Z","shell.execute_reply":"2025-05-13T20:35:17.408039Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame(\n    {\n        \"index\": test_df.index,\n        \"file_name\": test_df[\"file_name\"],\n        \"caption\": pred_captions,\n    }\n)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-13T20:35:17.409240Z","iopub.execute_input":"2025-05-13T20:35:17.409414Z","iopub.status.idle":"2025-05-13T20:35:17.425512Z","shell.execute_reply.started":"2025-05-13T20:35:17.409399Z","shell.execute_reply":"2025-05-13T20:35:17.424846Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## To keep only submission.csv file","metadata":{}},{"cell_type":"code","source":"!rm -rf /kaggle/working/embeddings_train\n!rm -rf /kaggle/working/embeddings_val\n!rm -rf /kaggle/working/embeddings_test","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}