{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-04T08:29:10.492519Z","iopub.execute_input":"2023-11-04T08:29:10.493299Z","iopub.status.idle":"2023-11-04T08:29:10.937381Z","shell.execute_reply.started":"2023-11-04T08:29:10.493257Z","shell.execute_reply":"2023-11-04T08:29:10.936373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pytorch-lightning","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:10.939331Z","iopub.execute_input":"2023-11-04T08:29:10.939732Z","iopub.status.idle":"2023-11-04T08:29:23.563468Z","shell.execute_reply.started":"2023-11-04T08:29:10.939706Z","shell.execute_reply":"2023-11-04T08:29:23.562517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport timm\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nimport torch\nfrom torch import optim, nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn import functional as F\nfrom torchvision import transforms as T\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom torchmetrics import Accuracy\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:23.565522Z","iopub.execute_input":"2023-11-04T08:29:23.565900Z","iopub.status.idle":"2023-11-04T08:29:38.124873Z","shell.execute_reply.started":"2023-11-04T08:29:23.565864Z","shell.execute_reply":"2023-11-04T08:29:38.123855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/digit-recognizer'\nsample_submission = pd.read_csv(f'{path}/sample_submission.csv')\ntrain = pd.read_csv(f'{path}/train.csv')\ntest = pd.read_csv(f'{path}/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:38.126080Z","iopub.execute_input":"2023-11-04T08:29:38.126667Z","iopub.status.idle":"2023-11-04T08:29:44.032703Z","shell.execute_reply.started":"2023-11-04T08:29:38.126638Z","shell.execute_reply":"2023-11-04T08:29:44.031710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.035187Z","iopub.execute_input":"2023-11-04T08:29:44.035986Z","iopub.status.idle":"2023-11-04T08:29:44.053431Z","shell.execute_reply.started":"2023-11-04T08:29:44.035959Z","shell.execute_reply":"2023-11-04T08:29:44.052234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.055043Z","iopub.execute_input":"2023-11-04T08:29:44.055369Z","iopub.status.idle":"2023-11-04T08:29:44.072476Z","shell.execute_reply.started":"2023-11-04T08:29:44.055338Z","shell.execute_reply":"2023-11-04T08:29:44.071485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.075593Z","iopub.execute_input":"2023-11-04T08:29:44.075865Z","iopub.status.idle":"2023-11-04T08:29:44.113902Z","shell.execute_reply.started":"2023-11-04T08:29:44.075843Z","shell.execute_reply":"2023-11-04T08:29:44.112950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitDataset(Dataset):\n    def __init__(self, df, train=True, transform=None):\n        self.train = train\n        self.transform = transform\n\n        if train:\n            self.X = df.drop(['label'], axis=1).values.astype('float32')\n            self.label = df['label'].values.tolist()\n        else:\n            self.X = df.values.astype('float32')\n    \n    def __len__(self):\n        return self.X.shape[0]\n    \n    def __getitem__(self, idx):\n        img = self.X[idx].reshape(28, 28)\n        if self.transform:\n            img = self.transform(img)\n\n        if not self.train:\n            return img\n        return img, self.label[idx]","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.115003Z","iopub.execute_input":"2023-11-04T08:29:44.115332Z","iopub.status.idle":"2023-11-04T08:29:44.127425Z","shell.execute_reply.started":"2023-11-04T08:29:44.115308Z","shell.execute_reply":"2023-11-04T08:29:44.126554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitModule(pl.LightningModule):\n    def __init__(self, model_name='resnet18', pretrained=True, num_classes=10):\n        super().__init__()\n        model = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=1,\n            num_classes=num_classes)\n        \"\"\"\n        for name, param in model.named_parameters():\n            param.requires_grad = False\n        m = list(model.children())\n        layers = m[:len(m):len(m)-1]\n        for layer in layers:\n            for param in layer.parameters():\n                param.requires_grad = True\n        \"\"\"\n        self.model = model\n\n        self.loss_fn = nn.CrossEntropyLoss()\n        self.accuracy = Accuracy(task='multiclass', num_classes=num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    def common(self, batch, batch_idx):\n        x, y = batch\n        scores = self(x)\n        loss = self.loss_fn(scores, y)\n        return loss, scores, y\n    \n    def training_step(self, batch, batch_idx):\n        loss, scores, y = self.common(batch, batch_idx)\n        self.log('train_loss', loss)\n        return loss\n        \n    def validation_step(self, batch, batch_idx):\n        loss, scores, y = self.common(batch, batch_idx)\n        y_pred = torch.argmax(scores, axis=1)\n        accuracy = self.accuracy(y_pred, y)\n        self.log_dict({\n            'validation_loss': loss,\n            'accuracy': accuracy,\n        })\n        return loss\n    \n    def test_step(self, batch, batch_idx):\n        loss, scores, y = self.common(batch, batch_idx)\n        self.log('test_loss', loss)\n        return loss\n    \n    def predict_step(self, batch, batch_idx):\n        x = batch\n        scores = self(x)\n        return scores\n    \n    def configure_optimizers(self):\n        optimizer = optim.Adam(self.parameters(), lr=0.001)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20, eta_min=0.001)\n        return [optimizer], [scheduler]","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.128622Z","iopub.execute_input":"2023-11-04T08:29:44.129474Z","iopub.status.idle":"2023-11-04T08:29:44.143875Z","shell.execute_reply.started":"2023-11-04T08:29:44.129428Z","shell.execute_reply":"2023-11-04T08:29:44.142799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitDataModule(pl.LightningDataModule):\n    def __init__(self, train_ds=None, val_ds=None, predict_ds=None, batch_size=32, num_workers=4):\n        super().__init__()\n        self.train_ds = train_ds\n        self.val_ds = val_ds\n        self.predict_ds = predict_ds\n        self.batch_size = batch_size   \n        self.num_workers = num_workers   \n                                             \n    def prepare_data(self):\n        pass\n                                             \n    def setup(self, stage):\n        pass\n                                                                                           \n    def train_dataloader(self):                                                            \n        return DataLoader(\n            self.train_ds,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=True,\n            pin_memory=True,\n        )\n                                             \n    def val_dataloader(self):\n        return DataLoader(                                                                 \n            self.val_ds,                                                                   \n            batch_size=self.batch_size,                                                                                                                                               \n            num_workers=self.num_workers,\n            shuffle=False, \n            pin_memory=True,\n        )\n    \n    def predict_dataloader(self):\n        return DataLoader(\n            self.predict_ds,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n            pin_memory=True,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.144998Z","iopub.execute_input":"2023-11-04T08:29:44.145624Z","iopub.status.idle":"2023-11-04T08:29:44.159312Z","shell.execute_reply.started":"2023-11-04T08:29:44.145593Z","shell.execute_reply":"2023-11-04T08:29:44.158488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128 * 32\nnum_fold = 6\nmax_epochs = 20\nnum_label = train['label'].nunique()\nmodel_name = 'resnet18'","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.160416Z","iopub.execute_input":"2023-11-04T08:29:44.161007Z","iopub.status.idle":"2023-11-04T08:29:44.178663Z","shell.execute_reply.started":"2023-11-04T08:29:44.160977Z","shell.execute_reply":"2023-11-04T08:29:44.177751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=num_fold, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.179774Z","iopub.execute_input":"2023-11-04T08:29:44.180034Z","iopub.status.idle":"2023-11-04T08:29:44.184812Z","shell.execute_reply.started":"2023-11-04T08:29:44.180012Z","shell.execute_reply":"2023-11-04T08:29:44.183888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_DIR = '.'","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.185820Z","iopub.execute_input":"2023-11-04T08:29:44.186203Z","iopub.status.idle":"2023-11-04T08:29:44.195187Z","shell.execute_reply.started":"2023-11-04T08:29:44.186180Z","shell.execute_reply":"2023-11-04T08:29:44.194349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = T.Compose([\n    T.ToTensor(),\n    T.RandomAffine(degrees=(-5, 5), translate=(0.025, 0.025), scale=(0.975, 1.025)),\n\n    #T.Normalize((0.5, ), (0.5, )),\n    T.Normalize((0.1307,), (0.3081,)),\n])\ntransform_test = T.Compose([\n    T.ToTensor(),\n    T.Normalize((0.1307,), (0.3081,)),\n])\n\npreds = []\nfor fold, (train_idx, test_idx) in enumerate(skf.split(train, train['label'])):\n    train_ds = DigitDataset(train.iloc[train_idx], train=True, transform=transform_train)\n    val_ds = DigitDataset(train.iloc[test_idx], train=True, transform=transform_test)\n    test_ds = DigitDataset(test, train=False, transform=transform_test)\n    model = DigitModule(model_name=model_name, pretrained=True, num_classes=num_label)\n    dm = DigitDataModule(\n        train_ds=train_ds,\n        val_ds=val_ds,\n        predict_ds=test_ds,\n        batch_size=batch_size,\n        num_workers=4\n    )\n    auc_checkpoint = ModelCheckpoint(\n        dirpath=OUTPUT_DIR,\n        filename=f'{model_name}_{fold}',\n        monitor='accuracy',\n        save_top_k=1,\n        save_weights_only=True,\n        mode='max',\n    )\n    trainer = pl.Trainer(\n        #accelerator='gpu',\n        callbacks=[auc_checkpoint],\n        default_root_dir=OUTPUT_DIR,\n        devices=1,\n        min_epochs=1,\n        max_epochs=max_epochs,\n        precision=16,\n    )\n    \n    trainer.fit(model, dm)\n    trainer.validate(model, dm)\n    pred = trainer.predict(model, dm)\n    preds.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:29:44.198838Z","iopub.execute_input":"2023-11-04T08:29:44.199719Z","iopub.status.idle":"2023-11-04T08:55:25.588964Z","shell.execute_reply.started":"2023-11-04T08:29:44.199686Z","shell.execute_reply":"2023-11-04T08:55:25.587105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(preds), len(preds[0]), preds[0][0].shape","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:55:25.594230Z","iopub.execute_input":"2023-11-04T08:55:25.594783Z","iopub.status.idle":"2023-11-04T08:55:25.607675Z","shell.execute_reply.started":"2023-11-04T08:55:25.594736Z","shell.execute_reply":"2023-11-04T08:55:25.606564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = torch.stack([torch.concat(pred) for pred in preds])","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:55:25.611327Z","iopub.execute_input":"2023-11-04T08:55:25.612480Z","iopub.status.idle":"2023-11-04T08:55:25.624004Z","shell.execute_reply.started":"2023-11-04T08:55:25.612430Z","shell.execute_reply":"2023-11-04T08:55:25.622922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = y_pred.mean(dim=0).argmax(dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:55:25.625965Z","iopub.execute_input":"2023-11-04T08:55:25.626602Z","iopub.status.idle":"2023-11-04T08:55:25.640290Z","shell.execute_reply.started":"2023-11-04T08:55:25.626573Z","shell.execute_reply":"2023-11-04T08:55:25.639220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['Label'] = y_pred\nsample_submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:55:25.641467Z","iopub.execute_input":"2023-11-04T08:55:25.641808Z","iopub.status.idle":"2023-11-04T08:55:25.708629Z","shell.execute_reply.started":"2023-11-04T08:55:25.641784Z","shell.execute_reply":"2023-11-04T08:55:25.707790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-04T08:55:25.709789Z","iopub.execute_input":"2023-11-04T08:55:25.710085Z","iopub.status.idle":"2023-11-04T08:55:25.719417Z","shell.execute_reply.started":"2023-11-04T08:55:25.710059Z","shell.execute_reply":"2023-11-04T08:55:25.718553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}