{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-03T00:16:22.480373Z","iopub.execute_input":"2023-11-03T00:16:22.481038Z","iopub.status.idle":"2023-11-03T00:16:22.489142Z","shell.execute_reply.started":"2023-11-03T00:16:22.481005Z","shell.execute_reply":"2023-11-03T00:16:22.488191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pytorch-lightning","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:22.490697Z","iopub.execute_input":"2023-11-03T00:16:22.490992Z","iopub.status.idle":"2023-11-03T00:16:34.160355Z","shell.execute_reply.started":"2023-11-03T00:16:22.490968Z","shell.execute_reply":"2023-11-03T00:16:34.159196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport timm\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nimport torch\nfrom torch import optim, nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn import functional as F\nfrom torchvision import transforms as T\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom torchmetrics import Accuracy\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:34.162840Z","iopub.execute_input":"2023-11-03T00:16:34.163236Z","iopub.status.idle":"2023-11-03T00:16:34.170443Z","shell.execute_reply.started":"2023-11-03T00:16:34.163200Z","shell.execute_reply":"2023-11-03T00:16:34.169446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/digit-recognizer'\nsample_submission = pd.read_csv(f'{path}/sample_submission.csv')\ntrain = pd.read_csv(f'{path}/train.csv')\ntest = pd.read_csv(f'{path}/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:34.171651Z","iopub.execute_input":"2023-11-03T00:16:34.171933Z","iopub.status.idle":"2023-11-03T00:16:38.743648Z","shell.execute_reply.started":"2023-11-03T00:16:34.171909Z","shell.execute_reply":"2023-11-03T00:16:38.742588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.746224Z","iopub.execute_input":"2023-11-03T00:16:38.746543Z","iopub.status.idle":"2023-11-03T00:16:38.758205Z","shell.execute_reply.started":"2023-11-03T00:16:38.746517Z","shell.execute_reply":"2023-11-03T00:16:38.757287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.759260Z","iopub.execute_input":"2023-11-03T00:16:38.759540Z","iopub.status.idle":"2023-11-03T00:16:38.780286Z","shell.execute_reply.started":"2023-11-03T00:16:38.759518Z","shell.execute_reply":"2023-11-03T00:16:38.779279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.781753Z","iopub.execute_input":"2023-11-03T00:16:38.782021Z","iopub.status.idle":"2023-11-03T00:16:38.801293Z","shell.execute_reply.started":"2023-11-03T00:16:38.781991Z","shell.execute_reply":"2023-11-03T00:16:38.800315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitDataset(Dataset):\n    def __init__(self, df, train=True, transform=None):\n        self.train = train\n        self.transform = transform\n\n        if train:\n            self.X = df.drop(['label'], axis=1).values.astype('float32')\n            self.label = df['label'].values.tolist()\n        else:\n            self.X = df.values.astype('float32')\n    \n    def __len__(self):\n        return self.X.shape[0]\n    \n    def __getitem__(self, idx):\n        img = self.X[idx].reshape(28, 28)\n        if self.transform:\n            img = self.transform(img)\n\n        if not self.train:\n            return img\n        return img, self.label[idx]","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.802352Z","iopub.execute_input":"2023-11-03T00:16:38.803588Z","iopub.status.idle":"2023-11-03T00:16:38.811758Z","shell.execute_reply.started":"2023-11-03T00:16:38.803560Z","shell.execute_reply":"2023-11-03T00:16:38.810896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitModule(pl.LightningModule):\n    def __init__(self, model_name='resnet18', pretrained=True, num_classes=10):\n        super().__init__()\n        model = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            in_chans=1,\n            num_classes=num_classes)\n        \"\"\"\n        for name, param in model.named_parameters():\n            param.requires_grad = False\n        m = list(model.children())\n        layers = m[:len(m):len(m)-1]\n        for layer in layers:\n            for param in layer.parameters():\n                param.requires_grad = True\n        \"\"\"\n        self.model = model\n\n        self.loss_fn = nn.CrossEntropyLoss()\n        self.accuracy = Accuracy(task='multiclass', num_classes=num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n    def common(self, batch, batch_idx):\n        x, y = batch\n        scores = self(x)\n        loss = self.loss_fn(scores, y)\n        return loss, scores, y\n    \n    def training_step(self, batch, batch_idx):\n        loss, scores, y = self.common(batch, batch_idx)\n        self.log('train_loss', loss)\n        return loss\n        \n    def validation_step(self, batch, batch_idx):\n        loss, scores, y = self.common(batch, batch_idx)\n        y_pred = torch.argmax(scores, axis=1)\n        accuracy = self.accuracy(y_pred, y)\n        self.log_dict({\n            'validation_loss': loss,\n            'accuracy': accuracy,\n        })\n        return loss\n    \n    def test_step(self, batch, batch_idx):\n        loss, scores, y = self.common(batch, batch_idx)\n        self.log('test_loss', loss)\n        return loss\n    \n    def predict_step(self, batch, batch_idx):\n        x = batch\n        scores = self(x)\n        return scores\n    \n    def configure_optimizers(self):\n        optimizer = optim.Adam(self.parameters(), lr=0.001)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20, eta_min=0.001)\n        return [optimizer], [scheduler]","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.813262Z","iopub.execute_input":"2023-11-03T00:16:38.813631Z","iopub.status.idle":"2023-11-03T00:16:38.826636Z","shell.execute_reply.started":"2023-11-03T00:16:38.813605Z","shell.execute_reply":"2023-11-03T00:16:38.825706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitDataModule(pl.LightningDataModule):\n    def __init__(self, train_ds=None, val_ds=None, predict_ds=None, batch_size=32, num_workers=4):\n        super().__init__()\n        self.train_ds = train_ds\n        self.val_ds = val_ds\n        self.predict_ds = predict_ds\n        self.batch_size = batch_size   \n        self.num_workers = num_workers   \n                                             \n    def prepare_data(self):\n        pass\n                                             \n    def setup(self, stage):\n        pass\n                                                                                           \n    def train_dataloader(self):                                                            \n        return DataLoader(\n            self.train_ds,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=True,\n            pin_memory=True,\n        )\n                                             \n    def val_dataloader(self):\n        return DataLoader(                                                                 \n            self.val_ds,                                                                   \n            batch_size=self.batch_size,                                                                                                                                               \n            num_workers=self.num_workers,\n            shuffle=False, \n            pin_memory=True,\n        )\n    \n    def predict_dataloader(self):\n        return DataLoader(\n            self.predict_ds,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n            pin_memory=True,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.827955Z","iopub.execute_input":"2023-11-03T00:16:38.828278Z","iopub.status.idle":"2023-11-03T00:16:38.840705Z","shell.execute_reply.started":"2023-11-03T00:16:38.828233Z","shell.execute_reply":"2023-11-03T00:16:38.839846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128 * 32\nnum_fold = 6\nmax_epochs = 20\nnum_label = train['label'].nunique()\nmodel_name = 'resnet18'","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.844110Z","iopub.execute_input":"2023-11-03T00:16:38.844378Z","iopub.status.idle":"2023-11-03T00:16:38.853854Z","shell.execute_reply.started":"2023-11-03T00:16:38.844355Z","shell.execute_reply":"2023-11-03T00:16:38.852971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=num_fold, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.856087Z","iopub.execute_input":"2023-11-03T00:16:38.856849Z","iopub.status.idle":"2023-11-03T00:16:38.862946Z","shell.execute_reply.started":"2023-11-03T00:16:38.856817Z","shell.execute_reply":"2023-11-03T00:16:38.861971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_DIR = '.'","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:16:38.864409Z","iopub.execute_input":"2023-11-03T00:16:38.864750Z","iopub.status.idle":"2023-11-03T00:16:38.872346Z","shell.execute_reply.started":"2023-11-03T00:16:38.864721Z","shell.execute_reply":"2023-11-03T00:16:38.871448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = T.Compose([\n    T.ToTensor(),\n    T.RandomAffine(degrees=(-5, 5), translate=(0.025, 0.025), scale=(0.975, 1.025)),\n\n    #T.Normalize((0.5, ), (0.5, )),\n    T.Normalize((0.1307,), (0.3081,)),\n])\ntransform_test = T.Compose([\n    T.ToTensor(),\n    T.Normalize((0.1307,), (0.3081,)),\n])\n\npreds = []\nfor fold, (train_idx, test_idx) in enumerate(skf.split(train, train['label'])):\n    train_ds = DigitDataset(train.iloc[train_idx], train=True, transform=transform_train)\n    val_ds = DigitDataset(train.iloc[test_idx], train=True, transform=transform_test)\n    test_ds = DigitDataset(test, train=False, transform=transform_test)\n    model = DigitModule(model_name=model_name, pretrained=True, num_classes=num_label)\n    dm = DigitDataModule(\n        train_ds=train_ds,\n        val_ds=val_ds,\n        predict_ds=test_ds,\n        batch_size=batch_size,\n        num_workers=4\n    )\n    auc_checkpoint = ModelCheckpoint(\n        dirpath=OUTPUT_DIR,\n        filename=f'{model_name}_{fold}',\n        monitor='accuracy',\n        save_top_k=1,\n        save_weights_only=True,\n        mode='max',\n    )\n    trainer = pl.Trainer(\n        #accelerator='gpu',\n        callbacks=[auc_checkpoint],\n        default_root_dir=OUTPUT_DIR,\n        devices=1,\n        min_epochs=1,\n        max_epochs=max_epochs,\n        precision=16,\n    )\n    \n    trainer.fit(model, dm)\n    trainer.validate(model, dm)\n    pred = trainer.predict(model, dm)\n    preds.append(pred)","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:39:52.572016Z","iopub.execute_input":"2023-11-03T00:39:52.572642Z","iopub.status.idle":"2023-11-03T00:42:26.644920Z","shell.execute_reply.started":"2023-11-03T00:39:52.572610Z","shell.execute_reply":"2023-11-03T00:42:26.643919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(preds), len(preds[0]), preds[0][0].shape","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:42:34.502560Z","iopub.execute_input":"2023-11-03T00:42:34.502930Z","iopub.status.idle":"2023-11-03T00:42:34.512867Z","shell.execute_reply.started":"2023-11-03T00:42:34.502894Z","shell.execute_reply":"2023-11-03T00:42:34.511848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = torch.stack([torch.concat(pred) for pred in preds])","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:42:35.202178Z","iopub.execute_input":"2023-11-03T00:42:35.203029Z","iopub.status.idle":"2023-11-03T00:42:35.216181Z","shell.execute_reply.started":"2023-11-03T00:42:35.202993Z","shell.execute_reply":"2023-11-03T00:42:35.215361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = y_pred.mean(dim=0).argmax(dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:42:35.621780Z","iopub.execute_input":"2023-11-03T00:42:35.622459Z","iopub.status.idle":"2023-11-03T00:42:35.630524Z","shell.execute_reply.started":"2023-11-03T00:42:35.622428Z","shell.execute_reply":"2023-11-03T00:42:35.629614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['Label'] = y_pred\nsample_submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:42:36.051882Z","iopub.execute_input":"2023-11-03T00:42:36.052719Z","iopub.status.idle":"2023-11-03T00:42:36.115155Z","shell.execute_reply.started":"2023-11-03T00:42:36.052682Z","shell.execute_reply":"2023-11-03T00:42:36.114405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:42:36.751720Z","iopub.execute_input":"2023-11-03T00:42:36.752408Z","iopub.status.idle":"2023-11-03T00:42:36.761725Z","shell.execute_reply.started":"2023-11-03T00:42:36.752367Z","shell.execute_reply":"2023-11-03T00:42:36.760735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}