{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":16017,"databundleVersionId":708189,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport configparser\nfrom pathlib import Path\nimport torch\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nimport gc\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\n #lots of imports.\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-26T15:23:59.892201Z","iopub.execute_input":"2024-04-26T15:23:59.893140Z","iopub.status.idle":"2024-04-26T15:23:59.899356Z","shell.execute_reply.started":"2024-04-26T15:23:59.893090Z","shell.execute_reply":"2024-04-26T15:23:59.898443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/Kannada-MNIST/train.csv')\ntarget = df_train['label']\ndf_train.drop('label', axis=1, inplace=True)\n\nX_test = pd.read_csv('../input/Kannada-MNIST/test.csv')\nX_test.drop('id', axis=1, inplace=True)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\nBATCH_SIZE = 256\nepoch = 100\n\nX_train, X_val, y_train, y_val = train_test_split(df_train, target, stratify=target, random_state=42, test_size=0.01)\n\ntransform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor()\n])\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:23:59.901112Z","iopub.execute_input":"2024-04-26T15:23:59.901384Z","iopub.status.idle":"2024-04-26T15:24:04.364490Z","shell.execute_reply.started":"2024-04-26T15:23:59.901360Z","shell.execute_reply":"2024-04-26T15:24:04.363469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n\n    def __init__(self,images,labels=None,transform=None,):\n        self.X = images\n        self.y = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        img = np.array(self.X.iloc[idx, :], dtype='uint8').reshape([28, 28, 1])\n        if self.transform is not None:\n            img = self.transform(img)\n        if self.y is not None:\n            y = np.zeros(10, dtype='float32')\n            y[self.y.iloc[idx]] = 1\n            return img, y\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.366159Z","iopub.execute_input":"2024-04-26T15:24:04.366546Z","iopub.status.idle":"2024-04-26T15:24:04.374309Z","shell.execute_reply.started":"2024-04-26T15:24:04.366510Z","shell.execute_reply":"2024-04-26T15:24:04.373389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CustomDataset(X_train, y_train, transform)\nval_dataset = CustomDataset(X_val, y_val, transform)\ntest_dataset = CustomDataset(X_test, transform=transform)\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\nval_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4) #workers 4 cuz kaggle is maxed at 4","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.376683Z","iopub.execute_input":"2024-04-26T15:24:04.377364Z","iopub.status.idle":"2024-04-26T15:24:04.393122Z","shell.execute_reply.started":"2024-04-26T15:24:04.377332Z","shell.execute_reply":"2024-04-26T15:24:04.392219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n#sometimes a simple model can give better results than one with extra techniques.\nclass Kmodel(nn.Module):\n    def __init__(self):\n        super(Kmodel, self).__init__()\n        \n        self.features = nn.Sequential(\n            nn.Conv2d(in_channels=1, out_channels=16, kernel_size=3, stride=1),\n            nn.ReLU(),\n            nn.Conv2d(in_channels=16, out_channels=64, kernel_size=3, stride=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2), #reduce image size by half for faster computation.\n            nn.Dropout(0.25) #dropouts against overfitting.\n        )\n        \n        self.classifier = nn.Sequential(\n            nn.Linear(12*12*64, 128),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(128, 10) #NO ACTIVATION FUNCTION because crossentropy loss takes pure logits as input.\n        )\n\n    def forward(self, x):\n        x = self.features(x)\n        x = torch.flatten(x, 1)\n        x = self.classifier(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.394279Z","iopub.execute_input":"2024-04-26T15:24:04.394542Z","iopub.status.idle":"2024-04-26T15:24:04.404803Z","shell.execute_reply.started":"2024-04-26T15:24:04.394518Z","shell.execute_reply":"2024-04-26T15:24:04.403812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Early stopper to detect overfitting, didnt use because kaggle gpu is slow and this would also slow things down a lot more\nclass EarlyStopper:\n    def __init__(self, patience=3, min_delta=0.003):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.min_validation_loss = float('inf')\n\n    def early_stop(self, validation_loss):\n        if validation_loss < self.min_validation_loss:\n            self.min_validation_loss = validation_loss\n            self.counter = 0\n        elif validation_loss > (self.min_validation_loss + self.min_delta):\n            self.counter += 1\n            if self.counter >= self.patience:\n                return True\n        return False\n#Essentially this looks at your current val loss, and the minimum and compares the two.\n#Delta is the offset, or the overhead, call it whatever you want, it basically gives your loss headroom and wont stop instantly\n#Patience is the amount of epochs your loss can be higher than the minimum before stopping the training.","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.405864Z","iopub.execute_input":"2024-04-26T15:24:04.406206Z","iopub.status.idle":"2024-04-26T15:24:04.418770Z","shell.execute_reply.started":"2024-04-26T15:24:04.406175Z","shell.execute_reply":"2024-04-26T15:24:04.417728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Kmodel()\nmodel=model.to(device) #send to gpu\ncriterion=torch.nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n#scheduler = lr_scheduler.StepLR(optimizer, step_size=60, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.420044Z","iopub.execute_input":"2024-04-26T15:24:04.420364Z","iopub.status.idle":"2024-04-26T15:24:04.614972Z","shell.execute_reply.started":"2024-04-26T15:24:04.420334Z","shell.execute_reply":"2024-04-26T15:24:04.613971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#Stopper=EarlyStopper()\ndef train(epoch,model):\n    model.train()\n    for epoch in range(epoch):\n        \n        loss=0\n        for batch_idx, (data, target) in enumerate(train_loader):\n            data = data.to(device)\n            target = target.to(device)\n\n            optimizer.zero_grad()\n            output = model(data)\n            loss = criterion(output, target)\n\n            loss.backward()\n            optimizer.step()\n\n            if (batch_idx + 1) % 100 == 0:\n                print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                    epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n                           100. * (batch_idx + 1) / len(train_loader),loss.data))\n        #THIS BELOW WOULD BE THE OVERFIT DETECTION, again decided to not use because its slow on kaggle plus model didnt overfit during training\n        '''\n        for batch_idx, (data, target) in enumerate(val_loader):\n            model.eval()\n            loss = 0\n            with torch.no_grad():\n                for data, target in val_loader:\n                    data = data.to(device)\n                    target = target.to(device)\n                    output = model(data)\n                    loss += criterion(output, target).data\n            loss /= len(val_loader.dataset)\n        print('current val loss:',loss.data)\n        if Stopper.early_stop(loss):\n            print(\"val loss stopped decreasing, exiting..\")\n            break\n        '''\n    #scheduler.step() scheduler resulted in a slightly lower private score","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.616366Z","iopub.execute_input":"2024-04-26T15:24:04.616741Z","iopub.status.idle":"2024-04-26T15:24:04.626255Z","shell.execute_reply.started":"2024-04-26T15:24:04.616705Z","shell.execute_reply":"2024-04-26T15:24:04.625159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(epoch,model)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:04.627444Z","iopub.execute_input":"2024-04-26T15:24:04.627817Z","iopub.status.idle":"2024-04-26T15:24:16.639926Z","shell.execute_reply.started":"2024-04-26T15:24:04.627785Z","shell.execute_reply":"2024-04-26T15:24:16.638029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model):\n    model.eval()\n    loss = 0\n    correct = 0\n\n    with torch.no_grad():\n        for data, target in val_loader:\n            data = data.to(device)\n            target = target.to(device)\n            output = model(data)\n            loss += criterion(output, target).data\n            pred = output.data.max(1, keepdim=True)[1]\n            correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()\n\n    loss /= len(val_loader.dataset)\n    accuracy = correct / len(val_loader.dataset)\n    print('Val loss: {:.4f}, Val accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(val_loader.dataset),\n        100. * accuracy))","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:16.641128Z","iopub.status.idle":"2024-04-26T15:24:16.641580Z","shell.execute_reply.started":"2024-04-26T15:24:16.641349Z","shell.execute_reply":"2024-04-26T15:24:16.641373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"evaluate(model)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:16.643236Z","iopub.status.idle":"2024-04-26T15:24:16.643690Z","shell.execute_reply.started":"2024-04-26T15:24:16.643450Z","shell.execute_reply":"2024-04-26T15:24:16.643468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\npredictions = []\n\nfor idx,data in enumerate(tqdm(test_loader)):\n    data = data.to(device)\n    output = model(data).max(dim=1)[1]\n    predictions += list(output.data.cpu().numpy())\n\nsubmission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission.to_csv('submission.csv', index=False)\n\nprint(submission.head)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T15:24:16.645291Z","iopub.status.idle":"2024-04-26T15:24:16.645598Z","shell.execute_reply.started":"2024-04-26T15:24:16.645447Z","shell.execute_reply":"2024-04-26T15:24:16.645459Z"},"trusted":true},"execution_count":null,"outputs":[]}]}