{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96745,"databundleVersionId":11513148,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport re\nimport gc\nimport platform\nimport random\nimport argparse\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\nimport einops\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nimport glob\nimport cv2\n\nfrom numpy import array\nfrom numpy import argmax\n\nfrom tqdm import tqdm\nfrom tqdm.contrib import tzip\n\nimport wandb\n\nimport warnings\n\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:16:41.179995Z","iopub.execute_input":"2025-04-16T08:16:41.180178Z","iopub.status.idle":"2025-04-16T08:16:54.88248Z","shell.execute_reply.started":"2025-04-16T08:16:41.180161Z","shell.execute_reply":"2025-04-16T08:16:54.88192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"conf = dict(\n    MAX_FRAMES=5,\n    EPOCHS=5,\n    LR=1e-4,\n    IMG_SIZE=(100, 100),\n    FEATURE_EXTRACTOR='convnext_base',\n    DR_RATE=0.2,\n    RNN_HIDDEN_SIZE=256,\n    RNN_LAYERS=1,\n    TRAIN_BS=10,\n    VALID_BS=10,\n    NUM_WORKERS=0,\n    MODEL_PATH='submit/weights/model_convnext_base.pt',\n    video_train_path='/kaggle/input/automated-video-captioning/train_videos',\n    video_test_path='/kaggle/input/automated-video-captioning/test_videos'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:13.962043Z","iopub.execute_input":"2025-04-16T08:17:13.962305Z","iopub.status.idle":"2025-04-16T08:17:13.966919Z","shell.execute_reply.started":"2025-04-16T08:17:13.962285Z","shell.execute_reply":"2025-04-16T08:17:13.96615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndf_train = pd.read_csv('/kaggle/input/automated-video-captioning/train.csv')\nsample_submit = pd.read_csv('/kaggle/input/automated-video-captioning/test.csv')\n\ndf_train.shape, sample_submit.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:15.910155Z","iopub.execute_input":"2025-04-16T08:17:15.910414Z","iopub.status.idle":"2025-04-16T08:17:15.934949Z","shell.execute_reply.started":"2025-04-16T08:17:15.910397Z","shell.execute_reply":"2025-04-16T08:17:15.934319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:17.360875Z","iopub.execute_input":"2025-04-16T08:17:17.361345Z","iopub.status.idle":"2025-04-16T08:17:17.380933Z","shell.execute_reply.started":"2025-04-16T08:17:17.361321Z","shell.execute_reply":"2025-04-16T08:17:17.380278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submit.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:19.512164Z","iopub.execute_input":"2025-04-16T08:17:19.512433Z","iopub.status.idle":"2025-04-16T08:17:19.520084Z","shell.execute_reply.started":"2025-04-16T08:17:19.512412Z","shell.execute_reply":"2025-04-16T08:17:19.519202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Количество размеченных видео в DataFrame: {len(set(df_train['file_name'].values))}\")\nprint(f\"Количество видео в папке train: {len(set(os.listdir('/kaggle/input/automated-video-captioning/train_videos')))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:21.227431Z","iopub.execute_input":"2025-04-16T08:17:21.227718Z","iopub.status.idle":"2025-04-16T08:17:21.241711Z","shell.execute_reply.started":"2025-04-16T08:17:21.227696Z","shell.execute_reply":"2025-04-16T08:17:21.241009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Размелим датасет на обучающую и валидационную выборку, т.е. train_test_split, только самописный\n\ndf_valid = df_train.sample(frac=0.2) #берем рандомно 20% от исходного датасета\ndf_train = df_train[~df_train.index.isin(df_valid.index)]# теперь удалим эти 20% из исходного, это и останется на трейн\n\ndf_train.reset_index(drop=True, inplace=True)\ndf_valid.reset_index(drop=True, inplace=True)\n\nprint(f'Train shape: {df_train.shape}')\nprint(f'Valid shape: {df_valid.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:22.831344Z","iopub.execute_input":"2025-04-16T08:17:22.831609Z","iopub.status.idle":"2025-04-16T08:17:22.845363Z","shell.execute_reply.started":"2025-04-16T08:17:22.831589Z","shell.execute_reply":"2025-04-16T08:17:22.844553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_video(path, img_size, transform=None, frames_num=11):\n    frames = []\n    cap = cv2.VideoCapture(path)\n\n    fps = int(cap.get(cv2.CAP_PROP_FPS))\n\n    length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    N = length // (frames_num)\n\n    current_frame = 1\n    for i in range(length):\n        ret, frame = cap.read(current_frame)\n\n        if ret and i == current_frame and len(frames) < frames_num:\n            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # все кадры в BGR (поэтому перводим в RGB)\n            frame = cv2.resize(frame, img_size)\n\n            frames.append(frame)\n            current_frame += N\n\n    cap.release()\n\n    return frames\n\nexample_path = '/kaggle/input/automated-video-captioning/test_videos/1.mp4'\nexample_video = read_video(path=example_path, img_size=conf['IMG_SIZE'], transform=None, frames_num=11)\nlen(example_video), example_video[0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:24.842838Z","iopub.execute_input":"2025-04-16T08:17:24.843112Z","iopub.status.idle":"2025-04-16T08:17:25.151396Z","shell.execute_reply.started":"2025-04-16T08:17:24.843091Z","shell.execute_reply":"2025-04-16T08:17:25.150742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nn = len(example_video)\ncols = 4  # Кол-во колонок в гриде\nrows = (n + cols - 1) // cols  # Вычисляем нужное кол-во строк\n\nplt.figure(figsize=(6, 5))\nfor i, img in enumerate(example_video):\n    plt.subplot(rows, cols, i + 1)\n    plt.imshow(img) \n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:29.033061Z","iopub.execute_input":"2025-04-16T08:17:29.033614Z","iopub.status.idle":"2025-04-16T08:17:29.409254Z","shell.execute_reply.started":"2025-04-16T08:17:29.033593Z","shell.execute_reply":"2025-04-16T08:17:29.408605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class VideoCaptionDataset(Dataset):\n    def __init__(self, df, conf, is_test=False):\n        self.df = df\n        self.conf = conf\n        self.is_test = is_test\n        self.samples = []\n\n        self.transform = A.Compose([\n            A.Normalize()\n        ])\n\n        base_path = conf['video_test_path'] if is_test else conf['video_train_path']\n\n        print(f\"{'Loading test' if is_test else 'Loading train'} dataset...\")\n\n        for idx, row in tqdm(df.iterrows(), total=len(df), desc=\"Reading videos\"):\n            video_path = os.path.join(base_path, row['file_name'])\n\n            frames = read_video(\n                path=video_path,\n                img_size=conf['IMG_SIZE'],\n                frames_num=conf['MAX_FRAMES']\n            )\n\n            if len(frames) == 0:\n                continue  # skip unreadable videos\n\n            sample = {'frames': frames}\n            if not is_test:\n                sample['caption'] = row['caption']\n\n            self.samples.append(sample)\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, idx):\n        sample = self.samples[idx]\n        frames = sample['frames']\n\n        video_tensor = torch.from_numpy(np.stack(frames)).permute(0, 3, 1, 2).float() / 255.\n\n        if self.is_test:\n            return video_tensor\n        else:\n            return video_tensor, sample['caption']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:17:33.465923Z","iopub.execute_input":"2025-04-16T08:17:33.466422Z","iopub.status.idle":"2025-04-16T08:17:33.474217Z","shell.execute_reply.started":"2025-04-16T08:17:33.4664Z","shell.execute_reply":"2025-04-16T08:17:33.473621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"# Create training dataset\ntrain_data = VideoCaptionDataset(df_train, conf, is_test=False)\n\ntrain_loader = DataLoader(\n    train_data,\n    batch_size=conf['TRAIN_BS'],\n    shuffle=True,  # shuffle for training!\n    num_workers=conf.get('NUM_WORKERS', 2)\n)\n\n# Create validation dataset\nvalid_data = VideoCaptionDataset(df_valid, conf, is_test=False)\n\nvalid_loader = DataLoader(\n    valid_data,\n    batch_size=conf['VALID_BS'],\n    shuffle=False,\n    num_workers=conf.get('NUM_WORKERS', 2)\n)\n\"\"\"\nprint(\"✅ DataLoaders ready!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:36:36.747107Z","iopub.execute_input":"2025-04-15T13:36:36.747605Z","iopub.status.idle":"2025-04-15T13:36:36.766069Z","shell.execute_reply.started":"2025-04-15T13:36:36.747584Z","shell.execute_reply":"2025-04-15T13:36:36.765392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"print(f'Sample len (это эквивалентно количеству видео): {len(train_data.samples)}')  # сколько видео в датасете\n\nprint(f'Что лежит в одном элементе: {len(train_data.samples[0])}')  # количество ключей в sample\nprint(f'Какие ключи есть: {train_data.samples[0].keys()}')  # названия ключей\n\nprint(f'Что лежит в ключе caption: {train_data.samples[0][\"caption\"]}')  # текстовая подпись\nprint(f'Количество кадров (frames) в видео: {len(train_data.samples[0][\"frames\"])}')  # число кадров\n\nprint(f'Размерность одного кадра (H, W, C): {train_data.samples[0][\"frames\"][0].shape}')  # размер одного кадра\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:36:36.766879Z","iopub.execute_input":"2025-04-15T13:36:36.767115Z","iopub.status.idle":"2025-04-15T13:36:36.784319Z","shell.execute_reply.started":"2025-04-15T13:36:36.767089Z","shell.execute_reply":"2025-04-15T13:36:36.78366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport timm\nfrom transformers import GPT2Tokenizer, GPT2LMHeadModel\nimport os\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nfrom nltk.translate.bleu_score import corpus_bleu\n\nclass ConvNextEncoder(nn.Module):\n    def __init__(self, conf):\n        super(ConvNextEncoder, self).__init__()\n        self.backbone = timm.create_model(conf['FEATURE_EXTRACTOR'], pretrained=True, num_classes=0)\n        self.feature_dim = self.backbone.num_features\n\n    def forward(self, x):\n        # x: [B * T, C, H, W]\n        return self.backbone(x)  # [B*T, F]\n\n\nimport os\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nfrom nltk.translate.bleu_score import corpus_bleu\n\nclass CaptioningModel(nn.Module):\n    def __init__(self, conf, tokenizer, prefix_length=5):\n        super(CaptioningModel, self).__init__()\n        from transformers import GPT2Tokenizer, GPT2LMHeadModel\n        import timm\n\n        self.conf = conf\n        self.tokenizer = tokenizer\n        self.prefix_length = prefix_length\n\n        # Visual encoder\n        self.encoder = timm.create_model(conf['FEATURE_EXTRACTOR'], pretrained=True, num_classes=0)\n        self.feature_dim = self.encoder.num_features\n\n        # GPT-2 decoder\n        self.gpt2 = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n        self.embedding_dim = self.gpt2.config.n_embd\n\n        # Bridge projects visual features → multiple GPT2 tokens\n        self.bridge = nn.Linear(self.feature_dim, self.embedding_dim * prefix_length)\n\n        # Optional output projection layer (trainable)\n        self.output_proj = nn.Linear(self.embedding_dim, self.gpt2.config.vocab_size, bias=False)\n\n    def forward(self, frames, captions_in):\n        B, T, C, H, W = frames.shape\n\n        x = frames.view(B * T, C, H, W)\n        features = self.encoder(x)              # [B*T, F]\n        features = features.view(B, T, -1)      # [B, T, F]\n        pooled = features.mean(dim=1)           # [B, F]\n\n        # Project to multiple GPT-2 embeddings\n        prefix_embed = self.bridge(pooled)                     # [B, E * prefix_length]\n        prefix_embed = prefix_embed.view(B, self.prefix_length, self.embedding_dim)  # [B, prefix_length, E]\n\n        # Embed the caption tokens\n        caption_embeddings = self.gpt2.transformer.wte(captions_in)  # [B, L, E]\n\n        # Concatenate visual prefix and text\n        gpt2_input = torch.cat([prefix_embed, caption_embeddings], dim=1)  # [B, prefix+L, E]\n\n        outputs = self.gpt2(inputs_embeds=gpt2_input)\n        logits = outputs.logits[:, self.prefix_length:, :]  # Drop visual prefix predictions\n        return logits\n\ndef main(conf, train_loader, valid_loader, tokenizer, output_path, device):\n    # Create model\n    model = CaptioningModel(conf, tokenizer=tokenizer)\n    model = model.to(device)\n\n    # Freeze all GPT-2 parameters except the bridge and output layers\n    for name, param in model.named_parameters():\n        if not any(k in name for k in ['bridge', 'output_proj']):\n            param.requires_grad = False\n\n    # Optimizer only for trainable parameters\n    optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=conf['LR'])\n\n    # Loss (ignore padding token)\n    loss_fn = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id)\n\n    print(f\"\\n📦 Using Backbone: {conf['FEATURE_EXTRACTOR']}, Decoder: GPT-2\")\n\n    best_val_loss = float('inf')\n\n    for epoch in range(conf['EPOCHS']):\n        print(f\"\\n{'--'*8} EPOCH: {epoch+1} {'--'*8}\\n\")\n\n        train_loss = train_one_epoch(\n            model,\n            train_loader,\n            optimizer,\n            loss_fn,\n            epoch,\n            device\n        )\n\n        val_loss, bleu = valid_one_epoch(\n            model,\n            valid_loader,\n            loss_fn,\n            epoch,\n            device,\n            tokenizer\n        )\n\n        print(f\"✅ Epoch {epoch+1} complete — val_loss: {val_loss:.4f}, BLEU: {bleu:.4f}\")\n\n        # Save the best model\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            torch.save(model.state_dict(), f\"model_{conf['FEATURE_EXTRACTOR']}_gpt2_finetune.pt\")\n            print(\"📁 Best model saved.\")\n\n    # Inference after training\n    print(\"\\n🔍 Generating predictions on validation set...\")\n    model.eval()\n    pred_texts = []\n    references = []\n\n    for frames, captions_in, targets in tqdm(valid_loader, desc=\"Inference\"):\n        frames = frames.to(device)\n        captions_in = captions_in.to(device)\n        targets = targets.to(device)\n\n        with torch.no_grad():\n            logits = model(frames, captions_in)\n            preds = logits.argmax(dim=-1)  # [B, L]\n\n        pred_texts += tokenizer.batch_decode(preds, skip_special_tokens=True)\n        references += tokenizer.batch_decode(targets, skip_special_tokens=True)\n\n    # Compute BLEU after full epoch\n    hypotheses = [pred.split() for pred in pred_texts]\n    refs = [[ref.split()] for ref in references]\n    final_bleu = corpus_bleu(refs, hypotheses)\n    print(f\"\\n🌟 Final BLEU score: {final_bleu:.4f}\")\n\n    # Save predictions to DataFrame\n    df_valid = valid_loader.dataset.df.copy()\n    df_valid['prediction'] = pred_texts\n    df_valid['target'] = references\n\n    os.makedirs(output_path, exist_ok=True)\n    df_valid.to_csv(os.path.join(output_path, \"predictions.csv\"), index=False)\n    print(\"📄 Saved predictions to:\", os.path.join(output_path, \"predictions.csv\"))\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:43:49.398161Z","iopub.execute_input":"2025-04-16T08:43:49.398849Z","iopub.status.idle":"2025-04-16T08:43:49.414781Z","shell.execute_reply.started":"2025-04-16T08:43:49.398822Z","shell.execute_reply":"2025-04-16T08:43:49.414026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk.translate.bleu_score import corpus_bleu\n\n\ndef train_one_epoch(model, train_dataloader, optimizer, loss_fn, epoch, device, verbose=False):\n    \"\"\"\n    Trains the GPT2-based captioning model for one epoch.\n\n    Args:\n        model: CaptioningModel\n        train_dataloader: DataLoader yielding (frames, input_ids, labels)\n        optimizer: optimizer for trainable parameters\n        loss_fn: loss function (e.g., CrossEntropyLoss with ignore_index)\n        epoch: current epoch number\n        device: 'cuda' or 'cpu'\n        verbose: if True, prints debug info\n\n    Returns:\n        Average loss over the epoch\n    \"\"\"\n    model.train()\n    running_loss = 0.0\n\n    prog_bar = tqdm(enumerate(train_dataloader), total=len(train_dataloader), desc=f\"Epoch {epoch}\")\n\n    for batch_idx, (frames, captions_in, targets) in prog_bar:\n        frames = frames.to(device, dtype=torch.float)\n        captions_in = captions_in.to(device)\n        targets = targets.to(device)\n\n        optimizer.zero_grad()\n\n        # Forward pass\n        logits = model(frames, captions_in)  # [B, L, vocab_size]\n\n        # Compute loss: flatten both for loss_fn\n        B, L, V = logits.shape\n        loss = loss_fn(logits.reshape(B * L, V), targets.reshape(B * L))\n\n        loss.backward()\n        optimizer.step()\n\n        loss_item = loss.item()\n        running_loss += loss_item\n\n        prog_bar.set_description(f\"Epoch {epoch} | Loss: {loss_item:.4f}\")\n\n        if verbose and batch_idx % 20 == 0:\n            print(f\"Batch {batch_idx} | Loss: {loss_item:.4f}\")\n\n    avg_loss = running_loss / len(train_dataloader)\n    return avg_loss\n\n\n@torch.no_grad()\ndef valid_one_epoch(model, valid_dataloader, loss_fn, epoch, device, tokenizer, verbose=False):\n    \"\"\"\n    Validates the GPT2-based captioning model for one epoch.\n\n    Args:\n        model: CaptioningModel\n        valid_dataloader: DataLoader yielding (frames, captions_in, targets)\n        loss_fn: loss function (e.g., CrossEntropyLoss with ignore_index)\n        epoch: current epoch number\n        device: 'cuda' or 'cpu'\n        tokenizer: tokenizer used for decoding captions\n        verbose: if True, prints debug info\n\n    Returns:\n        Average validation loss and BLEU score\n    \"\"\"\n    model.eval()\n    running_loss = 0.0\n    references = []\n    hypotheses = []\n\n    prog_bar = tqdm(enumerate(valid_dataloader), total=len(valid_dataloader), desc=f\"[Valid] Epoch {epoch}\")\n\n    for batch_idx, (frames, captions_in, targets) in prog_bar:\n        frames = frames.to(device, dtype=torch.float)\n        captions_in = captions_in.to(device)\n        targets = targets.to(device)\n\n        logits = model(frames, captions_in)  # [B, L, vocab_size]\n\n        B, L, V = logits.shape\n        loss = loss_fn(logits.reshape(B * L, V), targets.reshape(B * L))\n\n        loss_item = loss.item()\n        running_loss += loss_item\n\n        # Decode predictions and targets\n        pred_ids = logits.argmax(dim=-1)  # [B, L]\n        for i in range(B):\n            pred_tokens = tokenizer.decode(pred_ids[i], skip_special_tokens=True).split()\n            target_tokens = tokenizer.decode(targets[i], skip_special_tokens=True).split()\n            hypotheses.append(pred_tokens)\n            references.append([target_tokens])\n\n        prog_bar.set_description(f\"[Valid] Epoch {epoch} | Loss: {loss_item:.4f}\")\n\n        if verbose and batch_idx % 10 == 0:\n            print(f\"[Valid] Batch {batch_idx} | Loss: {loss_item:.4f}\")\n\n    avg_val_loss = running_loss / len(valid_dataloader)\n    bleu_score = corpus_bleu(references, hypotheses)\n    print(f\"[Valid] BLEU Score: {bleu_score:.4f}\")\n\n    return avg_val_loss, bleu_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T08:33:54.748093Z","iopub.execute_input":"2025-04-16T08:33:54.748615Z","iopub.status.idle":"2025-04-16T08:33:54.758908Z","shell.execute_reply.started":"2025-04-16T08:33:54.748593Z","shell.execute_reply":"2025-04-16T08:33:54.757999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def collate_fn(batch, tokenizer, max_len=20):\n    frames, captions = zip(*batch)  # unzip list of tuples\n\n    # Stack video tensors: [B, T, C, H, W]\n    frames = torch.stack(frames)\n    captions = [f\"{tokenizer.bos_token} {c} {tokenizer.eos_token}\" for c in captions]\n\n    # Tokenize with padding and truncation\n    tokenized = tokenizer(\n        captions,\n        padding='longest',       # or 'max_length'\n        max_length=max_len,\n        truncation=True,\n        return_tensors='pt',\n        add_special_tokens=False  # already added manually\n    )\n\n    input_ids = tokenized.input_ids  # [B, L]\n\n    # Shift to create captions_in and targets\n    captions_in = input_ids[:, :-1]   # e.g., [B, L-1]\n    targets = input_ids[:, 1:]        # e.g., [B, L-1]\n\n    return frames, captions_in, targets\n\ntokenizer = AutoTokenizer.from_pretrained(\"gpt2\")  # or gpt2, distilgpt2, etc.\ntokenizer.pad_token = tokenizer.eos_token  # important for GPT-style tokenizers\n\ntrain_dataset = VideoCaptionDataset(df_train, conf, is_test=False)\nvalid_dataset = VideoCaptionDataset(df_valid, conf, is_test=False)\n\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=conf['TRAIN_BS'],\n    shuffle=True,\n    num_workers=conf['NUM_WORKERS'],\n    collate_fn=lambda batch: collate_fn(batch, tokenizer)\n)\n\nvalid_loader = DataLoader(\n    valid_dataset,\n    batch_size=conf['VALID_BS'],\n    shuffle=False,\n    num_workers=conf['NUM_WORKERS'],\n    collate_fn=lambda batch: collate_fn(batch, tokenizer)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T09:03:01.546872Z","iopub.execute_input":"2025-04-16T09:03:01.547637Z","iopub.status.idle":"2025-04-16T09:05:30.175288Z","shell.execute_reply.started":"2025-04-16T09:03:01.547605Z","shell.execute_reply":"2025-04-16T09:05:30.174489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer\nfrom torch.utils.data import DataLoader\n\nconf = dict(\n    MAX_FRAMES=20,\n    EPOCHS=5,\n    LR=1e-5,\n    IMG_SIZE=(200, 200),\n    FEATURE_EXTRACTOR='convnext_base',\n    DR_RATE=0.2,\n    RNN_HIDDEN_SIZE=512,\n    RNN_LAYERS=2,\n    TRAIN_BS=5,\n    VALID_BS=5,\n    NUM_WORKERS=0,\n    MODEL_PATH='submit/weights/model_convnext_base.pt',\n    video_train_path='/kaggle/input/automated-video-captioning/train_videos',\n    video_test_path='/kaggle/input/automated-video-captioning/test_videos'\n)\n\n# === DEVICE ===\nimport platform\n\nif torch.cuda.is_available():\n    print(\"Using GPU:\", torch.cuda.get_device_name())\n    device = torch.device('cuda')\nelse:\n    print(\"Using CPU:\", platform.processor())\n    device = torch.device('cpu')\n\n# === TOKENIZER ===\ntokenizer = AutoTokenizer.from_pretrained(\"gpt2\")  # or gpt2, distilgpt2, etc.\ntokenizer.pad_token = tokenizer.eos_token  # important for GPT-style tokenizers\n\n# === RUN EVERYTHING ===\nmain(conf, train_loader, valid_loader, tokenizer, output_path='output/', device=device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T09:05:33.473868Z","iopub.execute_input":"2025-04-16T09:05:33.474143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom torch.utils.data import DataLoader\n\nsample_submit = pd.read_csv('/kaggle/input/automated-video-captioning/test.csv')\nsample_submit.columns = ['index', 'file_name']\nsample_submit['caption'] = ''  # empty placeholder\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:39:16.267677Z","iopub.status.idle":"2025-04-15T13:39:16.267912Z","shell.execute_reply.started":"2025-04-15T13:39:16.267794Z","shell.execute_reply":"2025-04-15T13:39:16.267811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"conf = dict(\n    MAX_FRAMES=5,\n    EPOCHS=5,\n    LR=1e-4,\n    IMG_SIZE=(100, 100),\n    FEATURE_EXTRACTOR='convnext_base',\n    DR_RATE=0.2,\n    RNN_HIDDEN_SIZE=256,\n    RNN_LAYERS=1,\n    TRAIN_BS=10,\n    VALID_BS=10,\n    NUM_WORKERS=0,\n    MODEL_PATH='submit/weights/model_convnext_base.pt',\n    video_train_path='/kaggle/input/automated-video-captioning/train_videos',\n    video_test_path='/kaggle/input/automated-video-captioning/test_videos'\n)\n\n\ntest_dataset = VideoCaptionDataset(sample_submit, conf, is_test=True)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=conf['VALID_BS'],\n    shuffle=False,\n    num_workers=conf['NUM_WORKERS']\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:39:16.269096Z","iopub.status.idle":"2025-04-15T13:39:16.269444Z","shell.execute_reply.started":"2025-04-15T13:39:16.269251Z","shell.execute_reply":"2025-04-15T13:39:16.269267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = CaptioningModel(conf, vocab_size=tokenizer.vocab_size, pad_idx=tokenizer.pad_token_id)\nmodel.load_state_dict(torch.load(f\"model_{conf['FEATURE_EXTRACTOR']}.pt\"))\nmodel = model.to(device)\nmodel.eval()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:39:16.270446Z","iopub.status.idle":"2025-04-15T13:39:16.270655Z","shell.execute_reply.started":"2025-04-15T13:39:16.270555Z","shell.execute_reply":"2025-04-15T13:39:16.270564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\npredictions = []\n\nfor frames in tqdm(test_loader, desc=\"Generating captions\"):\n    frames = frames.to(device).float()\n\n    with torch.no_grad():\n        # We repeat context vector, so we can fake a caption input of size L\n        L = 20  # max caption length you want\n        dummy_input = torch.full((frames.size(0), L), tokenizer.pad_token_id, dtype=torch.long).to(device)\n\n        logits = model(frames, dummy_input)\n        pred_ids = logits.argmax(dim=-1)  # [B, L]\n        decoded = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)\n        predictions.extend(decoded)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:39:16.271613Z","iopub.status.idle":"2025-04-15T13:39:16.272045Z","shell.execute_reply.started":"2025-04-15T13:39:16.271924Z","shell.execute_reply":"2025-04-15T13:39:16.271936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = sample_submit.copy()\nsubmission['caption'] = predictions\nsubmission[['index', 'file_name', 'caption']].to_csv('submission.csv', index=False)\nprint(\"✅ Saved submission.csv with\", len(submission), \"rows.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-15T13:39:16.272735Z","iopub.status.idle":"2025-04-15T13:39:16.272996Z","shell.execute_reply.started":"2025-04-15T13:39:16.272858Z","shell.execute_reply":"2025-04-15T13:39:16.272871Z"}},"outputs":[],"execution_count":null}]}