{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":129271,"databundleVersionId":15506307},{"sourceType":"datasetVersion","sourceId":14676503,"datasetId":9373645,"databundleVersionId":15519103}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## **Install HuggingFace Diarizers**\n\nThis cell clones the HuggingFace `diarizers` repository and installs it in editable mode so the training scripts can be modified and used directly within the notebook.","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/huggingface/diarizers.git","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%cd diarizers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:06:28.066577Z","iopub.execute_input":"2026-02-15T18:06:28.066853Z","iopub.status.idle":"2026-02-15T18:06:28.072406Z","shell.execute_reply.started":"2026-02-15T18:06:28.066824Z","shell.execute_reply":"2026-02-15T18:06:28.071694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -e .","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:06:28.073413Z","iopub.execute_input":"2026-02-15T18:06:28.073625Z","iopub.status.idle":"2026-02-15T18:07:19.975315Z","shell.execute_reply.started":"2026-02-15T18:06:28.073606Z","shell.execute_reply":"2026-02-15T18:07:19.974325Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Fix Dependency Versions for Compatibility**\n\nThis cell resets and reinstalls key libraries with compatible versions (notably `numpy<2.4`) to resolve conflicts between `pyannote`, `librosa`, `numba`, and HuggingFace packages, ensuring stable diarization training and inference.","metadata":{}},{"cell_type":"code","source":"!pip -q uninstall -y numpy numba llvmlite librosa huggingface-hub huggingface_hub datasets transformers tokenizers\n!pip -q install --no-cache-dir \"numpy<2.4\"\n!pip -q install --no-cache-dir \"numba\" \"llvmlite\" \"librosa\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:19.977692Z","iopub.execute_input":"2026-02-15T18:07:19.977967Z","iopub.status.idle":"2026-02-15T18:07:38.291693Z","shell.execute_reply.started":"2026-02-15T18:07:19.977939Z","shell.execute_reply":"2026-02-15T18:07:38.290948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /kaggle/working/constraints.txt\nnumpy<2.4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:38.293007Z","iopub.execute_input":"2026-02-15T18:07:38.293741Z","iopub.status.idle":"2026-02-15T18:07:38.298712Z","shell.execute_reply.started":"2026-02-15T18:07:38.293709Z","shell.execute_reply":"2026-02-15T18:07:38.298173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install -c /kaggle/working/constraints.txt --no-cache-dir \\\n  \"huggingface-hub==0.36.2\" \\\n  \"transformers==4.48.3\" \\\n  \"datasets==3.6.0\" \\\n  \"tokenizers==0.21.4\" \\\n  \"ffmpeg-python\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:38.299743Z","iopub.execute_input":"2026-02-15T18:07:38.300079Z","iopub.status.idle":"2026-02-15T18:07:50.785249Z","shell.execute_reply.started":"2026-02-15T18:07:38.300046Z","shell.execute_reply":"2026-02-15T18:07:50.784469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy\nprint(\"numpy:\", numpy.__version__)\n\nimport numba, librosa\nprint(\"numba:\", numba.__version__)\nprint(\"librosa:\", librosa.__version__)\n\nimport datasets, transformers\nprint(\"datasets:\", datasets.__version__)\nprint(\"transformers:\", transformers.__version__)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:50.786488Z","iopub.execute_input":"2026-02-15T18:07:50.786766Z","iopub.status.idle":"2026-02-15T18:07:52.174675Z","shell.execute_reply.started":"2026-02-15T18:07:50.786737Z","shell.execute_reply":"2026-02-15T18:07:52.174004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -y uninstall huggingface-hub huggingface_hub\n!pip -q install --no-cache-dir \"huggingface-hub==0.36.2\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:52.262752Z","iopub.execute_input":"2026-02-15T18:07:52.263018Z","iopub.status.idle":"2026-02-15T18:07:55.769087Z","shell.execute_reply.started":"2026-02-15T18:07:52.262987Z","shell.execute_reply":"2026-02-15T18:07:55.768101Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Configure HuggingFace Authentication**\n\nThis cell retrieves the HuggingFace access token from Kaggle Secrets, sets the required environment variables, and stores it in the local cache to enable authenticated model downloads.","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nimport os\nfrom pathlib import Path\n\nhf_token = UserSecretsClient().get_secret(\"HF_TOKEN\")\n\n# set env vars used by hub clients\nos.environ[\"HF_TOKEN\"] = hf_token\nos.environ[\"HUGGINGFACE_HUB_TOKEN\"] = hf_token\n\n# write token to HF cache (works even without huggingface_hub.login)\ntoken_path = Path.home() / \".cache\" / \"huggingface\" / \"token\"\ntoken_path.parent.mkdir(parents=True, exist_ok=True)\ntoken_path.write_text(hf_token)\n\nprint(\"HF token configured\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:55.770598Z","iopub.execute_input":"2026-02-15T18:07:55.771609Z","iopub.status.idle":"2026-02-15T18:07:56.214009Z","shell.execute_reply.started":"2026-02-15T18:07:55.771568Z","shell.execute_reply":"2026-02-15T18:07:56.213401Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Patch: Robust Dataset Loading**\n\nThis cell modifies `train_segmentation.py` to correctly handle both `load_from_disk()` (local HuggingFace datasets) and `load_dataset()` sources, ensuring compatibility with saved datasets and preventing split-related errors.","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport os\n\nfp = Path(\"/kaggle/working/diarizers/train_segmentation.py\")\nlines = fp.read_text().splitlines(True)  # keep newline chars\n\n\nfor i, l in enumerate(lines):\n    if l.strip().startswith(\"from datasets import\") and \"load_dataset\" in l:\n        # Make sure we have load_from_disk + Dataset + DatasetDict\n        lines[i] = \"from datasets import load_dataset, load_from_disk, Dataset, DatasetDict\\n\"\n        break\nelse:\n    raise RuntimeError(\"Couldn't find datasets import line to patch.\")\n\n\nstart = None\nend = None\nfor i, l in enumerate(lines):\n    if \"# Load the Dataset\" in l:\n        start = i\n        break\nif start is None:\n    raise RuntimeError(\"Couldn't find '# Load the Dataset' section.\")\n\nfor j in range(start + 1, len(lines)):\n    if \"train_split_name\" in lines[j]:\n        end = j\n        break\nif end is None:\n    raise RuntimeError(\"Couldn't find the end of the dataset load block (train_split_name line).\")\n\nnew_block = [\n\"    # Load the Dataset:\\n\",\n\"    dataset_name = str(data_args.dataset_name)\\n\",\n\"    if os.path.isdir(dataset_name) and (\\n\",\n\"        os.path.exists(os.path.join(dataset_name, 'dataset_info.json'))\\n\",\n\"        or os.path.exists(os.path.join(dataset_name, 'state.json'))\\n\",\n\"    ):\\n\",\n\"        dataset = load_from_disk(dataset_name)\\n\",\n\"        # If it's a single Dataset (no splits), wrap it so the rest of the script works\\n\",\n\"        if isinstance(dataset, Dataset):\\n\",\n\"            dataset = DatasetDict({'train': dataset})\\n\",\n\"    elif data_args.dataset_config_name:\\n\",\n\"        dataset = load_dataset(\\n\",\n\"            str(data_args.dataset_name),\\n\",\n\"            str(data_args.dataset_config_name),\\n\",\n\"            num_proc=int(data_args.preprocessing_num_workers)\\n\",\n\"        )\\n\",\n\"    else:\\n\",\n\"        dataset = load_dataset(\\n\",\n\"            str(data_args.dataset_name),\\n\",\n\"            num_proc=int(data_args.preprocessing_num_workers)\\n\",\n\"        )\\n\",\n\"\\n\",\n]\n\nlines[start:end] = new_block\nfp.write_text(\"\".join(lines))\nprint(\"✅ Patched:\", fp)\n\nprint(\"\\n--- Patched section preview ---\")\nfor k in range(start, min(start + 35, len(lines))):\n    print(f\"{k+1:4d}: {lines[k].rstrip()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:56.214966Z","iopub.execute_input":"2026-02-15T18:07:56.215225Z","iopub.status.idle":"2026-02-15T18:07:56.224331Z","shell.execute_reply.started":"2026-02-15T18:07:56.215203Z","shell.execute_reply":"2026-02-15T18:07:56.223624Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **Necessary imports**","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torchaudio\nimport transformers\nimport datasets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:07:56.225126Z","iopub.execute_input":"2026-02-15T18:07:56.225383Z","iopub.status.idle":"2026-02-15T18:07:58.042611Z","shell.execute_reply.started":"2026-02-15T18:07:56.225357Z","shell.execute_reply":"2026-02-15T18:07:58.041704Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Build HuggingFace Dataset for Diarization Training**\n\nThis cell converts the raw audio files and CSV annotations into a structured HuggingFace `Dataset`, creates train and validation splits, and saves it to disk for use with the diarization training script.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom datasets import Dataset, DatasetDict, Audio\n\naudio_dir = \"/kaggle/input/competitions/dl-sprint-4-0-bengali-speaker-diarization-challenge/diarization/diarization/train/audio\"\nann_dir   = \"/kaggle/input/competitions/dl-sprint-4-0-bengali-speaker-diarization-challenge/diarization/diarization/train/annotation\"\n\ndef time_to_seconds(t: str) -> float:\n    # supports \"HH:MM:SS\" or \"HH:MM:SS.mmm\"\n    h, m, s = t.strip().split(\":\")\n    return int(h)*3600 + int(m)*60 + float(s)\n\nrows = []\nfor wav in os.listdir(audio_dir):\n    if not wav.endswith(\".wav\"):\n        continue\n\n    rec = wav[:-4]\n    csv_path = os.path.join(ann_dir, rec + \".csv\")\n    if not os.path.exists(csv_path):\n        continue\n\n    df = pd.read_csv(csv_path)\n    df.columns = df.columns.str.strip().str.lower()\n\n    rows.append({\n        \"audio\": os.path.join(audio_dir, wav),\n        \"timestamps_start\": [time_to_seconds(x) for x in df[\"start_time\"]],\n        \"timestamps_end\":   [time_to_seconds(x) for x in df[\"end_time\"]],\n        \"speakers\":         [f\"speaker_{sid}\" for sid in df[\"speaker_id\"].astype(str).tolist()],\n    })\n\nds = Dataset.from_list(rows).cast_column(\"audio\", Audio(sampling_rate=16000))\n\nsplit = ds.train_test_split(test_size=0.1, seed=42)\ndataset = DatasetDict({\"train\": split[\"train\"], \"validation\": split[\"test\"]})\n\ndataset.save_to_disk(\"/kaggle/working/diar_dataset\")\ndataset\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:08:03.241063Z","iopub.execute_input":"2026-02-15T18:08:03.241718Z","iopub.status.idle":"2026-02-15T18:08:03.245466Z","shell.execute_reply.started":"2026-02-15T18:08:03.241693Z","shell.execute_reply":"2026-02-15T18:08:03.244644Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Build and Save Full Training Dataset**\n\nThis cell rebuilds the HuggingFace dataset from the raw audio and annotation CSVs, clears any previous saved version, and saves the complete dataset to `/kaggle/working/diar_dataset` for training without creating splits.","metadata":{}},{"cell_type":"code","source":"import os, shutil\nimport pandas as pd\nfrom datasets import Dataset, Audio\n\naudio_dir = \"/kaggle/input/competitions/dl-sprint-4-0-bengali-speaker-diarization-challenge/diarization/diarization/train/audio\"\nann_dir   = \"/kaggle/input/competitions/dl-sprint-4-0-bengali-speaker-diarization-challenge/diarization/diarization/train/annotation\"\n\nOUT = \"/kaggle/working/diar_dataset\"\nshutil.rmtree(OUT, ignore_errors=True)  \n\ndef time_to_seconds(t: str) -> float:\n    # supports HH:MM:SS or HH:MM:SS.mmm\n    h, m, s = t.strip().split(\":\")\n    return int(h)*3600 + int(m)*60 + float(s)\n\nrows = []\nfor wav in os.listdir(audio_dir):\n    if not wav.endswith(\".wav\"):\n        continue\n\n    rec = wav[:-4]\n    csv_path = os.path.join(ann_dir, rec + \".csv\")\n    if not os.path.exists(csv_path):\n        continue\n\n    df = pd.read_csv(csv_path)\n    df.columns = df.columns.str.strip().str.lower()\n\n    rows.append({\n        \"audio\": os.path.join(audio_dir, wav),\n        \"timestamps_start\": [time_to_seconds(x) for x in df[\"start_time\"]],\n        \"timestamps_end\":   [time_to_seconds(x) for x in df[\"end_time\"]],\n        \"speakers\":         [f\"speaker_{sid}\" for sid in df[\"speaker_id\"].astype(str).tolist()],\n    })\n\ndataset = Dataset.from_list(rows).cast_column(\"audio\", Audio(sampling_rate=16000))\ndataset.save_to_disk(OUT)\n\nprint(\"Saved:\", OUT)\nprint(\"Columns:\", dataset.column_names)\ndataset\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch, torchvision\nprint(torch.__version__, torchvision.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:08:03.548262Z","iopub.status.idle":"2026-02-15T18:08:03.548594Z","shell.execute_reply.started":"2026-02-15T18:08:03.548401Z","shell.execute_reply":"2026-02-15T18:08:03.548423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.environ[\"WANDB_DISABLED\"] = \"true\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:08:03.551778Z","iopub.status.idle":"2026-02-15T18:08:03.552009Z","shell.execute_reply.started":"2026-02-15T18:08:03.551901Z","shell.execute_reply":"2026-02-15T18:08:03.551914Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Fine-tune Segmentation Model**\n\nThis cell launches training for `pyannote/segmentation-3.0` on the prepared dataset and saves the fine-tuned checkpoint to `/kaggle/working/seg_finetuned`, with evaluation disabled to train on all available data.","metadata":{}},{"cell_type":"code","source":"!python /kaggle/working/diarizers/train_segmentation.py \\\n  --dataset_name \"/kaggle/working/diar_dataset\" \\\n  --split_on_subset train \\\n  --model_name_or_path pyannote/segmentation-3.0 \\\n  --output_dir \"/kaggle/working/seg_finetuned\" \\\n  --do_train \\\n  --do_eval \\\n  --learning_rate 5e-4 \\\n  --per_device_train_batch_size 8 \\\n  --per_device_eval_batch_size 8 \\\n  --gradient_accumulation_steps 2 \\\n  --num_train_epochs 12 \\\n  --lr_scheduler_type cosine \\\n  --warmup_ratio 0.1 \\\n  --optim adamw_torch \\\n  --seed 42 \\\n  --fp16 \\\n  --preprocessing_num_workers 1 \\\n  --report_to none","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T18:08:03.553145Z","iopub.status.idle":"2026-02-15T18:08:03.553501Z","shell.execute_reply.started":"2026-02-15T18:08:03.553325Z","shell.execute_reply":"2026-02-15T18:08:03.553347Z"}},"outputs":[],"execution_count":null}]}