{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4753077,"sourceType":"datasetVersion","datasetId":2750746},{"sourceId":13095143,"sourceType":"datasetVersion","datasetId":8294140},{"sourceId":13498377,"sourceType":"datasetVersion","datasetId":8570474},{"sourceId":278364861,"sourceType":"kernelVersion"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:05.541249Z","iopub.execute_input":"2025-11-16T07:58:05.541444Z","iopub.status.idle":"2025-11-16T07:58:05.915226Z","shell.execute_reply.started":"2025-11-16T07:58:05.541426Z","shell.execute_reply":"2025-11-16T07:58:05.914391Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataloader","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/inatsounds-annots/augmented_annot_rare_sep.csv')\ndf['supercategory_id'] -= 1\ndf['name_id'] -= 1\ndf_2 = df[df['supercategory_id']==1].copy()\n\n\nval_df = pd.read_csv(\"/kaggle/input/inatsounds-annots/val_annotation.csv\")\nval_df['supercategory_id'] -= 1\nval_df['name_id'] -= 1\nval_df = val_df[val_df['supercategory_id']==1].copy()\n\n\nrare_df = pd.read_csv(\"/kaggle/input/inatsounds-annots/rare_test.csv\")\nrare_df['supercategory_id'] -= 1\nrare_df['name_id'] -= 1\nrare_df = rare_df[rare_df['supercategory_id']==1].copy()\n# df = pd.read_csv('/kaggle/input/sound-of-114-species-of-birds-till-2022/Birds Voice.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:05.916494Z","iopub.execute_input":"2025-11-16T07:58:05.916902Z","iopub.status.idle":"2025-11-16T07:58:07.066983Z","shell.execute_reply.started":"2025-11-16T07:58:05.916875Z","shell.execute_reply":"2025-11-16T07:58:07.066151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"super_cat_df = {\n    0: 3846,\n    1: 32,\n    2: 296,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:07.067724Z","iopub.execute_input":"2025-11-16T07:58:07.067983Z","iopub.status.idle":"2025-11-16T07:58:07.072335Z","shell.execute_reply.started":"2025-11-16T07:58:07.067960Z","shell.execute_reply":"2025-11-16T07:58:07.071540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nimport torchaudio\nimport torchaudio.transforms as T\nimport torch\nfrom IPython.display import clear_output\nimport librosa ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:07.074353Z","iopub.execute_input":"2025-11-16T07:58:07.074709Z","iopub.status.idle":"2025-11-16T07:58:08.929209Z","shell.execute_reply.started":"2025-11-16T07:58:07.074684Z","shell.execute_reply":"2025-11-16T07:58:08.928268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:08.929982Z","iopub.execute_input":"2025-11-16T07:58:08.930464Z","iopub.status.idle":"2025-11-16T07:58:08.975534Z","shell.execute_reply.started":"2025-11-16T07:58:08.930434Z","shell.execute_reply":"2025-11-16T07:58:08.974615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ex, sr = torchaudio.load(\"/kaggle/input/sound-of-114-species-of-birds-till-2022/Voice of Birds/Voice of Birds/Australian Brushturkey_sound/Australian Brushturkey11.mp3\")\nex = ex.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:08.976399Z","iopub.execute_input":"2025-11-16T07:58:08.976718Z","iopub.status.idle":"2025-11-16T07:58:09.115159Z","shell.execute_reply.started":"2025-11-16T07:58:08.976686Z","shell.execute_reply":"2025-11-16T07:58:09.114323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GetData(Dataset):\n    def __init__(self, annot_file, split, stage, aug=True):\n        \n        self.annot_file = annot_file.copy().reset_index()\n        self.label_1 = self.annot_file['supercategory_id']\n        self.label_2 = self.annot_file['name_id']\n        self.file_dir = split\n        self.sr = 22050\n        self.stage = stage\n        self.aug = aug\n    \n    def __len__(self):\n        return len(self.label_1)\n\n    def pitch_shift(self, audio):\n        shift = np.random.uniform(low=-3, high=3)\n\n        return lr.effects.pitch_shift(y=audio, sr=self.sr, n_steps=shift)\n\n    def time_stretch(self, audio):\n        shift = np.random.uniform(low=0.6, high=1.3)\n\n        return lr.effects.time_stretch(y=audio, rate=shift)\n\n    def gain(self, audio):\n        peak_amplitude = np.max(np.abs(audio))\n        if peak_amplitude == 0:\n            return audio\n        \n        normalized_audio = audio / peak_amplitude\n        gain_factor = np.random.uniform(low=0.7, high=1.3)\n        \n        return normalized_audio * gain_factor\n    \n    def __getitem__(self, idx):\n        file_dir = self.annot_file.loc[idx, 'file_name']\n        if self.file_dir==\"train\": \n            aug_type = self.annot_file.loc[idx, 'aug_type']\n        aug_func = {\n            'Pitch Shift': self.pitch_shift,\n            'Time Stretch': self.time_stretch,\n            'Gain': self.gain\n        }\n        try:\n            audio, _ = librosa.load(f'/kaggle/input/i-nat-sounds/{self.file_dir}/{file_dir}')\n\n            if self.file_dir==\"train\" and aug_type in ['Pitch Shift', 'Time Stretch', 'Gain']:\n                if self.aug:\n                    audio = aug_func[aug_type](audio)\n\n            if self.stage == 1:\n                label = self.label_1.loc[idx] \n            else:\n                label = self.label_2.loc[idx] \n            return torch.from_numpy(audio).float(), torch.tensor(label)\n       \n        except:\n            rand_idx = np.random.randint(0, len(self))\n            return self.__getitem__(rand_idx)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.116217Z","iopub.execute_input":"2025-11-16T07:58:09.116497Z","iopub.status.idle":"2025-11-16T07:58:09.125528Z","shell.execute_reply.started":"2025-11-16T07:58:09.116475Z","shell.execute_reply":"2025-11-16T07:58:09.124692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sorted_unique_ids = sorted(df_2['name_id'].unique())\nlocal_id_map = {global_id: i for i, global_id in enumerate(sorted_unique_ids)}\n\ndf_2['name_id'] = df_2['name_id'].map(local_id_map)\nval_df['name_id'] = val_df['name_id'].map(local_id_map)\nrare_df['name_id'] = rare_df['name_id'].map(local_id_map)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.126194Z","iopub.execute_input":"2025-11-16T07:58:09.126435Z","iopub.status.idle":"2025-11-16T07:58:09.147297Z","shell.execute_reply.started":"2025-11-16T07:58:09.126412Z","shell.execute_reply":"2025-11-16T07:58:09.146754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_2['name_id'].max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.147949Z","iopub.execute_input":"2025-11-16T07:58:09.148182Z","iopub.status.idle":"2025-11-16T07:58:09.166284Z","shell.execute_reply.started":"2025-11-16T07:58:09.148155Z","shell.execute_reply":"2025-11-16T07:58:09.165572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = GetData(df_2, 'train', 2)\nval_data = GetData(val_df, 'val', 2)\nrare_data = GetData(rare_df, 'train', 2, aug=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.168981Z","iopub.execute_input":"2025-11-16T07:58:09.169254Z","iopub.status.idle":"2025-11-16T07:58:09.181287Z","shell.execute_reply.started":"2025-11-16T07:58:09.169238Z","shell.execute_reply":"2025-11-16T07:58:09.180722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_counts = torch.bincount(torch.tensor(df_2['name_id'].values))\nweights = 1.0/class_counts\nsampler = WeightedRandomSampler(weights, num_samples=len(train_data), replacement=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.182068Z","iopub.execute_input":"2025-11-16T07:58:09.182730Z","iopub.status.idle":"2025-11-16T07:58:09.191868Z","shell.execute_reply.started":"2025-11-16T07:58:09.182705Z","shell.execute_reply":"2025-11-16T07:58:09.191378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = DataLoader(train_data, batch_size=1, sampler=sampler)\nval_loader = DataLoader(val_data, batch_size=1)\nrare_loader = DataLoader(rare_data, batch_size=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.192708Z","iopub.execute_input":"2025-11-16T07:58:09.192939Z","iopub.status.idle":"2025-11-16T07:58:09.205350Z","shell.execute_reply.started":"2025-11-16T07:58:09.192918Z","shell.execute_reply":"2025-11-16T07:58:09.204830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"spectrogram_transform = T.MelSpectrogram(\n        n_fft=2048,  # Number of FFT points\n        hop_length=512, # Number of samples between successive frames\n        win_length=2048, # Window size\n        n_mels=128,\n        power=2.0,  # Power of the spectrogram (e.g., 2.0 for power spectrogram)\n        )\ndb_transform = T.AmplitudeToDB()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.205916Z","iopub.execute_input":"2025-11-16T07:58:09.206113Z","iopub.status.idle":"2025-11-16T07:58:09.223956Z","shell.execute_reply.started":"2025-11-16T07:58:09.206099Z","shell.execute_reply":"2025-11-16T07:58:09.223232Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Implementation","metadata":{}},{"cell_type":"code","source":"from torch import nn\nimport torch\nimport torchaudio.transforms as T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.224611Z","iopub.execute_input":"2025-11-16T07:58:09.224798Z","iopub.status.idle":"2025-11-16T07:58:09.238424Z","shell.execute_reply.started":"2025-11-16T07:58:09.224783Z","shell.execute_reply":"2025-11-16T07:58:09.237902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Spectrogram tower","metadata":{}},{"cell_type":"code","source":"class Spectrogram6x3(nn.Module):\n    def __init__(self):\n        super(Spectrogram6x3, self).__init__()\n\n        self.seq = nn.Sequential(\n            nn.Conv2d(in_channels=1, out_channels=256, kernel_size=(6, 3), padding=(3, 0), stride=3), nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n\n    def forward(self, x):\n        return self.seq(x)\n\nclass Spectrogram3x6(nn.Module):\n    def __init__(self):\n        super(Spectrogram3x6, self).__init__()\n\n        self.seq = nn.Sequential(\n            nn.Conv2d(in_channels=1, out_channels=256, kernel_size=(3, 6), padding=(0, 3), stride=3), nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n\n    def forward(self, x):\n        return self.seq(x)\n\nclass SpectrogramTower(nn.Module):\n    def __init__(self):\n        super(SpectrogramTower, self).__init__()\n\n        \n        self.model6x3 = Spectrogram6x3()\n        self.model3x6 = Spectrogram3x6()\n        \n        self.squash = nn.Sequential(\n            nn.BatchNorm2d(512),\n            nn.Conv2d(in_channels=512, out_channels=256, kernel_size=(22, 1)), nn.ReLU(),\n        )\n    def forward(self, x):\n    \n        # print('after')\n        # print(x.shape)\n        output6x3 = self.model6x3(x)\n        output3x6 = self.model3x6(x)\n        # print(output3x6.shape)\n        # print(output6x3.shape)\n        stacked =  torch.cat((output6x3, output3x6), dim=0)\n        # print(stacked.shape)\n        stacked = torch.unsqueeze(stacked, 0)\n        # print(stacked.shape)\n        output = self.squash(stacked)\n        output = torch.squeeze(output, dim=2)\n        # print(\"Spec\", output.shape)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.239109Z","iopub.execute_input":"2025-11-16T07:58:09.239299Z","iopub.status.idle":"2025-11-16T07:58:09.253349Z","shell.execute_reply.started":"2025-11-16T07:58:09.239284Z","shell.execute_reply":"2025-11-16T07:58:09.252694Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Raw Tower","metadata":{}},{"cell_type":"code","source":"class RawTowerX3(nn.Module):\n    def __init__(self):\n        super(RawTowerX3, self).__init__()\n        self.seq = nn.Sequential(\n            nn.Conv1d(in_channels=1, out_channels=256, kernel_size=3, stride=3), nn.ReLU(),\n            nn.MaxPool1d(kernel_size=2, stride=2)\n        )        \n\n    def forward(self, x):\n        return self.seq(x)    \n\nclass RawTowerX6(nn.Module):\n    def __init__(self):\n        super(RawTowerX6, self).__init__()\n        self.seq = nn.Sequential(\n            nn.Conv1d(in_channels=1, out_channels=256, kernel_size=6, stride=3, padding=3), nn.ReLU(),\n            nn.MaxPool1d(kernel_size=2, stride=2)\n        )        \n\n    def forward(self, x):\n        return self.seq(x)    \n\nclass RawTower(nn.Module):\n    def __init__(self):\n        super(RawTower, self).__init__()\n        self.modelx3 = RawTowerX3()\n        self.modelx6 = RawTowerX6()\n        self.squeeze = nn.Sequential(\n            nn.BatchNorm1d(512),\n            nn.Conv1d(in_channels=512, out_channels=256, kernel_size=512, stride=512), nn.ReLU(),\n        )\n\n    def forward(self, x):\n\n        outputX3 = self.modelx3(x)\n        outputX6 = self.modelx6(x)\n        stacked =  torch.cat((outputX6, outputX3), dim=0)\n        stacked = torch.unsqueeze(stacked, 0)\n        output = self.squeeze(stacked)\n        # print(\"Raw\", output.shape)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.254100Z","iopub.execute_input":"2025-11-16T07:58:09.254364Z","iopub.status.idle":"2025-11-16T07:58:09.271697Z","shell.execute_reply.started":"2025-11-16T07:58:09.254344Z","shell.execute_reply":"2025-11-16T07:58:09.271125Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## FFT Tower","metadata":{}},{"cell_type":"code","source":"class FFT_TowerX3(nn.Module):\n    def __init__(self):\n        super(FFT_TowerX3, self).__init__()\n        self.seq = nn.Sequential(\n            nn.Conv1d(in_channels=1, out_channels=256, kernel_size=3, stride=3), nn.ReLU(),\n            nn.MaxPool1d(kernel_size=2, stride=2)\n        )        \n\n    def forward(self, x):\n        return self.seq(x)    \n\nclass FFT_TowerX6(nn.Module):\n    def __init__(self):\n        super(FFT_TowerX6, self).__init__()\n        self.seq = nn.Sequential(\n            nn.Conv1d(in_channels=1, out_channels=256, kernel_size=6, stride=3, padding=3), nn.ReLU(),\n        )        \n\n    def forward(self, x):\n        return self.seq(x)    \n\nclass FFT_Tower(nn.Module):\n    def __init__(self):\n        super(FFT_Tower, self).__init__()\n        \n        self.modelx3 = FFT_TowerX3()\n        self.modelx6 = FFT_TowerX6()\n        \n        self.squeeze = nn.Sequential(\n            nn.BatchNorm1d(512),\n            nn.Conv1d(in_channels=512, out_channels=256, kernel_size=512, stride=512), nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n\n\n    def forward(self, x):\n        outputX3 = self.modelx3(x)\n        outputX6 = self.modelx6(x)\n        stacked =  torch.cat((outputX6, outputX3), dim=0)\n        stacked = torch.unsqueeze(stacked, 0)\n        output = self.squeeze(stacked)\n        # print(\"FFT\", output.shape)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.272324Z","iopub.execute_input":"2025-11-16T07:58:09.272558Z","iopub.status.idle":"2025-11-16T07:58:09.289589Z","shell.execute_reply.started":"2025-11-16T07:58:09.272532Z","shell.execute_reply":"2025-11-16T07:58:09.288894Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Augur","metadata":{}},{"cell_type":"code","source":"class Augur1(nn.Module):\n    def __init__(self, out_channels=3):\n        super(Augur1, self).__init__()\n\n        self.spec_tower = SpectrogramTower()\n\n        self.seq = nn.Sequential(\n            nn.Dropout1d(p=0.2),\n            nn.Conv1d(in_channels=64, out_channels=2, kernel_size=1),\n            nn.AdaptiveAvgPool1d(1),\n            nn.Conv1d(in_channels=2, out_channels=out_channels, kernel_size=1),\n        )\n    def forward(self, x):\n\n    \n        spec_output = self.spec_tower(x)\n        output = self.seq(spec_output)\n        # print(\"Augur\", output.shape)\n            \n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.290261Z","iopub.execute_input":"2025-11-16T07:58:09.290526Z","iopub.status.idle":"2025-11-16T07:58:09.305437Z","shell.execute_reply.started":"2025-11-16T07:58:09.290504Z","shell.execute_reply":"2025-11-16T07:58:09.304900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Augur2(nn.Module):\n    def __init__(self, out_channels=4174):\n        super(Augur2, self).__init__()\n\n        self.spec_tower = SpectrogramTower()\n        self.raw_tower = RawTower()\n        # self.fft_tower = FFT_Tower()\n\n        self.seq = nn.Sequential(\n            nn.Conv1d(in_channels=512, out_channels=512, kernel_size=1), nn.ReLU(),\n            nn.Dropout1d(p=0.3),\n            nn.AdaptiveAvgPool1d(1),\n            nn.Conv1d(in_channels=512, out_channels=out_channels, kernel_size=1)\n        )\n\n    def forward(self, raw, spec):\n\n        # print(\"Augur\", raw.shape)\n        \n        spec_output = self.spec_tower(spec)\n        raw_output = self.raw_tower(raw)\n        # fft_output = self.fft_tower(fft)\n        if spec_output.shape[2] != raw_output.shape[2]:\n            pad_need = spec_output.shape[2] - raw_output.shape[2]\n            raw_output = nn.functional.pad(raw_output, pad=(0, pad_need))\n            # fft_output = nn.functional.pad(fft_output, pad=(0, pad_need))\n\n        stacked =  torch.cat((spec_output, raw_output), dim=1)\n        # print(\"Augur\")\n        # print(stacked.shape)\n\n        output = self.seq(stacked)\n        # print(\"Augur\", output.shape)\n        \n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.306069Z","iopub.execute_input":"2025-11-16T07:58:09.306289Z","iopub.status.idle":"2025-11-16T07:58:09.320371Z","shell.execute_reply.started":"2025-11-16T07:58:09.306273Z","shell.execute_reply":"2025-11-16T07:58:09.319710Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn.functional as F","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:59:44.917607Z","iopub.execute_input":"2025-11-16T07:59:44.918323Z","iopub.status.idle":"2025-11-16T07:59:44.921677Z","shell.execute_reply.started":"2025-11-16T07:59:44.918302Z","shell.execute_reply":"2025-11-16T07:59:44.920890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=None, gamma=2.0, reduction='mean', ignore_index=-100):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha # Can be a float or a tensor for class-specific weights\n        self.gamma = gamma\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n\n    def forward(self, inputs, targets):\n        # Calculate cross-entropy loss\n        ce_loss = F.cross_entropy(inputs, targets, reduction='none', ignore_index=self.ignore_index)\n        \n        # Calculate probabilities for the true class\n        pt = torch.exp(-ce_loss)\n\n        # Apply focusing parameter\n        focal_term = (1 - pt) ** self.gamma\n        loss = focal_term * ce_loss\n\n        # Apply alpha weighting if provided\n        if self.alpha is not None:\n            if isinstance(self.alpha, (float, int)):\n                alpha_t = self.alpha\n            else: # Assuming alpha is a tensor of class weights\n                alpha_t = self.alpha.gather(0, targets.view(-1))\n            loss = alpha_t * loss\n\n        # Apply reduction\n        if self.reduction == 'mean':\n            return loss.mean()\n        elif self.reduction == 'sum':\n            return loss.sum()\n        else:\n            return loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:59:49.428152Z","iopub.execute_input":"2025-11-16T07:59:49.428627Z","iopub.status.idle":"2025-11-16T07:59:49.434787Z","shell.execute_reply.started":"2025-11-16T07:59:49.428603Z","shell.execute_reply":"2025-11-16T07:59:49.433933Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pre-Processing","metadata":{}},{"cell_type":"code","source":"def get_mel_spec(signal):\n    # print(type(signal))\n    # librosa.feature\n    x = librosa.feature.melspectrogram(y=signal, sr=22050,\n                                  n_fft=2052,\n                                  hop_length=516,\n                                   win_length=2052,\n                                   n_mels=132,\n                                   power=2.0)\n    # print(x.shape)\n    x = torch.from_numpy(x)\n    if x.shape[-1]%6 != 0:\n        pad_need = 6 - (x.shape[-1]%6)\n        # print(pad_need)\n        x = nn.functional.pad(x, pad=(0, pad_need))\n    return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.339108Z","iopub.execute_input":"2025-11-16T07:58:09.339323Z","iopub.status.idle":"2025-11-16T07:58:09.357463Z","shell.execute_reply.started":"2025-11-16T07:58:09.339309Z","shell.execute_reply":"2025-11-16T07:58:09.356890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_fft(signal):\n    fft_numpy = np.fft.fft(signal)\n    fft = torch.from_numpy(fft_numpy)\n    return torch.abs(fft)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.358187Z","iopub.execute_input":"2025-11-16T07:58:09.358387Z","iopub.status.idle":"2025-11-16T07:58:09.372688Z","shell.execute_reply.started":"2025-11-16T07:58:09.358365Z","shell.execute_reply":"2025-11-16T07:58:09.372028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_ = Augur2().to(device)\n# model_(ex).shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.373432Z","iopub.execute_input":"2025-11-16T07:58:09.373675Z","iopub.status.idle":"2025-11-16T07:58:09.385595Z","shell.execute_reply.started":"2025-11-16T07:58:09.373653Z","shell.execute_reply":"2025-11-16T07:58:09.385093Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Loop","metadata":{}},{"cell_type":"code","source":"class Dict_Optim():\n    def __init__(self, model_dict, lr):\n        self.optim = dict()\n        for key in model_dict.keys():\n            self.optim[key] = torch.optim.Adam(model_dict[key].parameters(), lr=lr)\n\n    def zero_grad(self):\n        for key in self.optim.keys():\n            self.optim[key].zero_grad()\n\n    def step(self, unique_keys):\n        for key in unique_keys:\n            if key in self.optim:\n                self.optim[key].step()            ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.386247Z","iopub.execute_input":"2025-11-16T07:58:09.386491Z","iopub.status.idle":"2025-11-16T07:58:09.402669Z","shell.execute_reply.started":"2025-11-16T07:58:09.386475Z","shell.execute_reply":"2025-11-16T07:58:09.402108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.403334Z","iopub.execute_input":"2025-11-16T07:58:09.403503Z","iopub.status.idle":"2025-11-16T07:58:09.417659Z","shell.execute_reply.started":"2025-11-16T07:58:09.403490Z","shell.execute_reply":"2025-11-16T07:58:09.416899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate(model, loss_fn, dataloader):\n    correct, total = 0, 0\n    losses = []\n    model.eval()\n    \n    with torch.no_grad():\n        # for _ in range(9657):\n        for i, (inputs, labels) in enumerate(dataloader):\n            labels = torch.tensor([[labels]]).to(device)\n            x = inputs.detach().numpy()\n            x_length = x.shape[1]\n            x_rem = x_length % 6\n            if x_rem > 0:\n                # Trimming the raw input on device\n                x = x[:,:x_length - x_rem]\n            inputs_spec = get_mel_spec(x).to(device)\n            # inputs_fft = calculate_fft(x).to(device)\n            x = torch.from_numpy(x).to(device)\n            outputs = model(x, inputs_spec).to(device)\n            del inputs, x, inputs_spec\n            loss = loss_fn(outputs, labels)\n            losses.append(loss.cpu().item())\n            _, predicted = torch.max(outputs.data, 1)\n            # print(predicted, labels)\n            total += labels.size(0)\n            correct += (predicted.to(device) == labels).sum().item()\n            # print(i+1)\n            # if (i+1) % 1000 == 0:\n                # print(100 * correct / total)\n    accuracy = correct / total\n    # print(f\"Accuracy: {accuracy}%\")\n    model.train()\n    return accuracy, np.mean(losses)\n# evaluate(model, val_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T08:05:49.917406Z","iopub.execute_input":"2025-11-16T08:05:49.917994Z","iopub.status.idle":"2025-11-16T08:05:49.924256Z","shell.execute_reply.started":"2025-11-16T08:05:49.917970Z","shell.execute_reply":"2025-11-16T08:05:49.923487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_loop(model, lr, loss_fn, batch_size, super_id=None):\n    # Ensure all models are on their assigned device outside this function!\n    # model on device0, model_2 experts and Feature Extractor on device1\n\n    optim = torch.optim.Adam(model.parameters(), lr=lr)\n\n    \n    expert_keys_used = []\n    losses = [0]\n    val_losses = [0]\n    rare_losses = [0]\n    accuracies = [0]\n    val_accuracies = [0]\n    rare_accuracies = [0]\n    best_val_loss = float('inf')\n    patience_counter = 0        \n    PATIENCE = 10\n    PATIENCE_FOR_LR_DECAY = 5\n    best_model_state = None\n    MIN_DELTA = 0.00005\n    \n    for epoch in range(6):\n        print(\"Current Epoch:\", epoch+1)\n        running_loss = 0.0\n        correct, total = 0, 0\n        optim.zero_grad()\n        expert_keys_used = []\n        for i, (inputs, labels) in enumerate(train_loader):\n            # 1. Inputs (raw audio) and Experts/Features stay on device1\n            # 2. Router (Model 1) and its labels go to device0\n            inputs_raw = inputs.squeeze(1).to(\"cpu\") \n            labels = torch.tensor([[labels]]).to(device)\n            if inputs_raw.shape[1] < 22050:\n                print(f\"Skipping short audio sample at batch {i}. Length: {inputs_raw.shape[1]}\")\n                continue\n            # --- Input Preprocessing (on device) ---\n            x = inputs_raw.detach().numpy()\n            x_length = x.shape[1]\n            x_rem = x_length % 6\n            if x_rem > 0:\n                # Trimming the raw input on device\n                x = x[:,:x_length - x_rem]\n            \n            # --- Feature Calculation ONCE (on device for memory) ---\n            # NOTE: Assumes get_mel_spec/calculate_fft logic is now on device\n            with torch.no_grad(): # No need to track gradients for this shared feature map\n                input_spec = get_mel_spec(x).to(device)\n                # input_fft = calculate_fft(x).to(device)\n                x = torch.from_numpy(x).to(device)\n\n            if super_id: \n                output = model(x, input_spec)\n            else: \n                output = model(x)\n            # Loss 1 calculated on device\n            loss = loss_fn(output, labels) / batch_size\n            \n            # Tracking model before deleting all variables for spcae\n            prob = torch.softmax(output.to('cpu'), dim=1)\n            predicted = torch.argmax(prob, dim=1).squeeze().item()\n            total += labels.size(0)\n            correct += (1 if predicted == labels.item() else 0)\n            running_loss += loss.item() * batch_size\n            \n            del x, input_spec\n            del labels, output\n            loss.backward()\n            del loss\n            if (i+1) % batch_size == 0:\n                optim.step()\n                # unique_keys = list(set(expert_keys_used))\n                \n                # 3. ZERO GRADIENTS: Reset both optimizers together\n                optim.zero_grad()\n                torch.cuda.empty_cache()\n\n\n            # --- Model 1 Tracking ---\n       \n                avg_loss = running_loss / (batch_size*1.5)\n                running_loss = 0.0\n                acc = correct / total\n                # print(f'{i/100}: {acc}')\n                correct, total = 0, 0\n                losses.append(avg_loss)\n                accuracies.append(acc)\n                print(f\"Step {int((i+1)/batch_size)} Done | ACC : {acc} | Loss : {avg_loss}\", flush=True) # Added flush=True for immediate log\n                \n                val_accuracy, val_loss = evaluate(model, loss_fn, val_loader)\n                val_losses.append(val_loss)\n                val_accuracies.append(val_accuracy)\n                print(f\"Val Acc: {val_accuracy} | Loss: {val_loss}\")\n                \n                rare_accuracy, rare_loss = evaluate(model, loss_fn, rare_loader)\n                rare_losses.append(rare_loss)\n                rare_accuracies.append(rare_accuracy)\n                print(f\"Rare Acc: {rare_accuracy} | Loss : {rare_loss}\")\n                del val_accuracy, val_loss, rare_accuracy, rare_loss, acc, avg_loss\n                # Realtime Plot\n                clear_output(wait=True)\n                plt.clf()\n\n                fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 8))\n                # print(val_losses, val_accuracies)\n                # print(accuracies_2)\n                ax1.plot(losses, label='Training Loss')\n                ax1.plot(val_losses, label='Val Loss')\n                ax1.plot(rare_losses, label='Rare Rare')\n                ax1.set_title(f\"Epoch {epoch+1}\")\n                ax1.set_xlabel(f\"{batch_size}-Batch Steps\")\n                ax1.set_ylabel(\"Loss Value\")\n                ax1.legend()\n                ax1.grid(True)\n\n                # Plot Accuracies\n                # print(accuracies)\n                # break\n                ax2.plot(accuracies, label='Accuracy')\n                ax2.plot(val_accuracies, label='Val Accuracy')\n                ax2.plot(rare_accuracies, label='Rare Accuracy')\n                ax2.set_title(f\"Epoch {epoch+1}\")\n                ax2.set_xlabel(f\"{batch_size}-Batch Steps\")\n                ax2.set_ylabel(\"Loss\")\n                ax2.legend()\n                ax2.grid(True)\n                \n                plt.tight_layout()\n                plt.show()\n                plt.pause(0.001)\n        print(i)\n        optim.step()\n\n\n        # print(\"Average:\", sum(accuracies)/len(accuracies))\n            # if i == 15:\n                # break\n    \n    #print('Last Accuracy:', acc)  \n   # print('Last Loss:', avg_loss)  \n    print('Finished Training')\n    plt.ioff()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T08:09:51.663021Z","iopub.execute_input":"2025-11-16T08:09:51.663587Z","iopub.status.idle":"2025-11-16T08:09:51.678010Z","shell.execute_reply.started":"2025-11-16T08:09:51.663566Z","shell.execute_reply":"2025-11-16T08:09:51.677113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_loader.dataset.annot_file['name_id'].max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.450379Z","iopub.execute_input":"2025-11-16T07:58:09.450780Z","iopub.status.idle":"2025-11-16T07:58:09.465656Z","shell.execute_reply.started":"2025-11-16T07:58:09.450764Z","shell.execute_reply":"2025-11-16T07:58:09.465171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_counts = torch.bincount(torch.tensor(df_2['name_id'].values))\nweights = 1.0/class_counts\n# Move the weights tensor to the device where the loss calculation will happen (your 'device')\nweights_tensor = weights.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T08:04:48.720462Z","iopub.execute_input":"2025-11-16T08:04:48.721082Z","iopub.status.idle":"2025-11-16T08:04:48.725837Z","shell.execute_reply.started":"2025-11-16T08:04:48.721059Z","shell.execute_reply":"2025-11-16T08:04:48.725171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:09.479874Z","iopub.execute_input":"2025-11-16T07:58:09.480160Z","iopub.status.idle":"2025-11-16T07:58:09.491212Z","shell.execute_reply.started":"2025-11-16T07:58:09.480138Z","shell.execute_reply":"2025-11-16T07:58:09.490580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# state_dict = torch.load('/kaggle/input/augur-c-1/model_weights-1-1.pth', weights_only=True)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nlr = 1e-6\nloss_fn = FocalLoss(alpha=None, gamma=2.0, reduction=None)\nstage = 1\nepochs = 5\nmodel = Augur2(32).to(device)\n# model.load_state_dict(state_dict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T08:09:55.167744Z","iopub.execute_input":"2025-11-16T08:09:55.168388Z","iopub.status.idle":"2025-11-16T08:09:55.822169Z","shell.execute_reply.started":"2025-11-16T08:09:55.168362Z","shell.execute_reply":"2025-11-16T08:09:55.821585Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# next(model.parameters()).device","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:58:10.166174Z","iopub.execute_input":"2025-11-16T07:58:10.166748Z","iopub.status.idle":"2025-11-16T07:58:10.170601Z","shell.execute_reply.started":"2025-11-16T07:58:10.166718Z","shell.execute_reply":"2025-11-16T07:58:10.169762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loop(model, lr=lr, loss_fn=loss_fn, super_id=1, batch_size=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T08:09:57.492838Z","iopub.execute_input":"2025-11-16T08:09:57.493595Z","iopub.status.idle":"2025-11-16T08:12:26.336516Z","shell.execute_reply.started":"2025-11-16T08:09:57.493572Z","shell.execute_reply":"2025-11-16T08:12:26.335377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH = f\"model_weights-{stage}-1.pth\"\n# Save the router model (model_1)\ntorch.save(model.state_dict(), PATH)\n\n# Save each expert model in model_2 separately\n# # for key, model in model_2.items(): # Corrected from .itmes\n#     torch.save(model.state_dict(), f\"model_weight-{key}.pth\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_df = pd.read_csv(\"/kaggle/input/inatsounds-annots/test_split_2.csv\")\nval_df['supercategory_id'] -= 1\nval_df['name_id'] -= 1\nval_df = val_df[val_df['supercategory_id']==1]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_df['name_id'] = val_df['name_id'].map(local_id_map)\n# rare_df['name_id'] = rare_df['name_id'].map(local_id_map)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# rare_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_data = GetData(val_df, 'test', 2)\nval_loader = DataLoader(val_data, batch_size=1)\n# rare_data = GetData(rare_df, 'train', 2, aug=False)\n# rare_loader = DataLoader(rare_data, batch_size=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy, loss = evaluate(model, loss_fn, val_loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# loss","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_2['name_id'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# val_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_2[df_2['name_id']==11].head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# x_spec = get_mel_spec(x_test)\n# x_fft = calculate_fft(x_test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# accuracies = evaluate(model, val_loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}