{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3362,"databundleVersionId":31148,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import transforms,models\nfrom torch.utils.data import Dataset,DataLoader\nfrom IPython.display import clear_output\nfrom PIL import Image\n\ndevice='cuda:0' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-22T13:32:16.911589Z","iopub.execute_input":"2024-01-22T13:32:16.911882Z","iopub.status.idle":"2024-01-22T13:32:20.895346Z","shell.execute_reply.started":"2024-01-22T13:32:16.911857Z","shell.execute_reply":"2024-01-22T13:32:20.894310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -q /kaggle/input/dogs-vs-cats/train.zip -d /kaggle/working\nlen(os.listdir('/kaggle/working/train'))","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:32:20.897314Z","iopub.execute_input":"2024-01-22T13:32:20.897854Z","iopub.status.idle":"2024-01-22T13:32:33.326005Z","shell.execute_reply.started":"2024-01-22T13:32:20.897814Z","shell.execute_reply":"2024-01-22T13:32:33.324909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self,path,transform,test=False,start=0,end=-1):\n        self.test=test\n        self.path=path\n        self.files=os.listdir(self.path)[start:end]\n        \n        if not self.test:\n            self.labels=torch.zeros(len(self.files))\n            for i,f in enumerate(self.files):\n                if 'cat' in f:\n                    self.labels[i]=1\n                elif 'dog' in f:\n                    self.labels[i]=0\n        self.transforms=transform\n    def __len__(self):\n        return len(self.files)\n    def __getitem__(self,idx):\n        img=Image.open(os.path.join(self.path,self.files[idx]))\n        img.load()\n        img=self.transforms(img)\n        if not self.test:\n            return img,self.labels[idx]\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:02:58.473573Z","iopub.execute_input":"2024-01-22T14:02:58.473951Z","iopub.status.idle":"2024-01-22T14:02:58.483294Z","shell.execute_reply.started":"2024-01-22T14:02:58.473922Z","shell.execute_reply":"2024-01-22T14:02:58.482209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install pytorch-lightning\n!pip install torchmetrics lightning timm\n!pip install tbparse\n!python -m pip install git+https://github.com/lessw2020/Ranger21.git\n# clear_output()\nimport ranger21\nimport timm\nimport lightning as L\nimport torchmetrics","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:32:33.341114Z","iopub.execute_input":"2024-01-22T13:32:33.341581Z","iopub.status.idle":"2024-01-22T13:33:24.753991Z","shell.execute_reply.started":"2024-01-22T13:32:33.341553Z","shell.execute_reply":"2024-01-22T13:33:24.752901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files=os.listdir('/kaggle/working/train')\n[len([f for f in files if category in f]) for category in ['dog','cat']]","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:24.755211Z","iopub.execute_input":"2024-01-22T13:33:24.755676Z","iopub.status.idle":"2024-01-22T13:33:24.782227Z","shell.execute_reply.started":"2024-01-22T13:33:24.755647Z","shell.execute_reply":"2024-01-22T13:33:24.781278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Lit(L.LightningModule):\n    def __init__(self,model,exp_name='try0',\\\n                 crit=nn.CrossEntropyLoss(),\\\n                 opt_class=optim.AdamW,\n                 opt_kwargs={'lr':1e-3}):\n        super().__init__()\n        self.model=model\n        self.crit=crit\n        self.opt_class=opt_class\n        self.opt_kwargs=opt_kwargs\n        \n        self.trn_metric=torchmetrics.Accuracy(task='binary')\n        self.val_metric=torchmetrics.Accuracy(task='binary')\n\n       \n        self.trn_metric=self.trn_metric.to('cpu')\n        self.val_metric=self.val_metric.to('cpu')\n\n    def configure_optimizers(self):\n        opt=self.opt_class(self.model.parameters(),**self.opt_kwargs)\n        return opt\n    \n    def training_step(self,batch,idx):\n        X,y=batch\n        y_pred=self.model(X.to(device)).flatten()\n        loss=self.crit(y_pred,y.to(device))\n        self.trn_metric.update(y_pred,y)\n        self.log('train_loss',loss.detach().item())\n        return loss\n    def validation_step(self,batch,idx):\n        X,y=batch\n        with torch.no_grad():\n            y_pred=self.model(X.to(device)).flatten()\n            loss=self.crit(y_pred,y.to(device))\n            \n            self.val_metric.update(y_pred,y)\n            self.log('val_loss',loss.detach().item())\n        return loss\n    def on_train_epoch_end(self):\n        metric=self.trn_metric.compute()\n        print(f'trn accuracy:{metric}')\n        self.log('train_metric',metric)\n        self.trn_metric.reset()\n        \n    def on_validation_epoch_end(self):\n        metric=self.val_metric.compute()\n        print(f'val accuracy:{metric}')\n        self.log('val_metric',metric)\n        self.val_metric.reset\n        \n    def test_step(self,batch,idx):\n        X,y=batch\n        with torch.no_grad():\n            y_pred=self.model(X)\n\n            acc=self.valid_acc(y_pred.detach().cpu(),y.cpu())\n\n            self.log_dict({'test_acc': acc})","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:24.783552Z","iopub.execute_input":"2024-01-22T13:33:24.784230Z","iopub.status.idle":"2024-01-22T13:33:24.798152Z","shell.execute_reply.started":"2024-01-22T13:33:24.784194Z","shell.execute_reply":"2024-01-22T13:33:24.797255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trn_ds,val_ds=torch.utils.data.random_split(ds,[20000,5000])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:24.799400Z","iopub.execute_input":"2024-01-22T13:33:24.799771Z","iopub.status.idle":"2024-01-22T13:33:24.813241Z","shell.execute_reply.started":"2024-01-22T13:33:24.799723Z","shell.execute_reply":"2024-01-22T13:33:24.812471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torchvision import models\ntimm.list_models('*efficientnet_b0*',pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:24.814215Z","iopub.execute_input":"2024-01-22T13:33:24.814566Z","iopub.status.idle":"2024-01-22T13:33:24.828882Z","shell.execute_reply.started":"2024-01-22T13:33:24.814542Z","shell.execute_reply":"2024-01-22T13:33:24.827988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model=timm.create_model('tf_efficientnet_b0.in1k')\nmodel=models.efficientnet_b0(weights='IMAGENET1K_V1')\nmodel.classifier=nn.Sequential(\n    nn.Linear(in_features=1280, out_features=1, bias=True),\n    nn.Sigmoid()\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:24.830107Z","iopub.execute_input":"2024-01-22T13:33:24.830788Z","iopub.status.idle":"2024-01-22T13:33:25.380303Z","shell.execute_reply.started":"2024-01-22T13:33:24.830761Z","shell.execute_reply":"2024-01-22T13:33:25.379286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_config = timm.data.resolve_model_data_config(model)\nim_size=data_config['input_size'][1:]\nmean=data_config['mean']\nstd =data_config['std']\n\ntrn_transform=transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=mean,std=std),\n    transforms.RandomResizedCrop(im_size,antialias=False),\n    transforms.RandomRotation(180)\n])\n\nval_transform=transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=mean,std=std),\n    transforms.Resize(im_size,antialias=False)\n])\n\ntrn_ds=CatsDogsDataset('/kaggle/working/train',trn_transform,start=0,end=20000)\nval_ds=CatsDogsDataset('/kaggle/working/train',val_transform,start=20000,end=25000)\n\ntrain_loader=DataLoader(trn_ds,batch_size=32,shuffle=True,num_workers=2)\nval_loader=DataLoader(val_ds,batch_size=32,shuffle=False,num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:25.382875Z","iopub.execute_input":"2024-01-22T13:33:25.383198Z","iopub.status.idle":"2024-01-22T13:33:25.542186Z","shell.execute_reply.started":"2024-01-22T13:33:25.383171Z","shell.execute_reply":"2024-01-22T13:33:25.541091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lit_model = Lit(model,crit=nn.BCELoss(),\\\n               opt_class=ranger21.Ranger21,\n               opt_kwargs={'lr':1e-3,'num_epochs':10,\\\n                            'num_batches_per_epoch':len(train_loader)})\ntrainer = L.Trainer(max_epochs=10, log_every_n_steps=5)\ntrainer.fit(model=lit_model, train_dataloaders=train_loader, val_dataloaders=val_loader)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:36:22.523106Z","iopub.execute_input":"2024-01-22T13:36:22.524010Z","iopub.status.idle":"2024-01-22T14:01:05.910056Z","shell.execute_reply.started":"2024-01-22T13:36:22.523972Z","shell.execute_reply":"2024-01-22T14:01:05.909167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tbparse import SummaryReader\n\nlog_dir = trainer.logger.experiment.get_logdir()  # lightning_logs/version_x\n\nreader = SummaryReader(log_dir, pivot=True)\ndf = reader.scalars\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:01:08.849201Z","iopub.execute_input":"2024-01-22T14:01:08.850133Z","iopub.status.idle":"2024-01-22T14:01:09.367129Z","shell.execute_reply.started":"2024-01-22T14:01:08.850094Z","shell.execute_reply":"2024-01-22T14:01:09.366101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig=plt.figure(figsize=(8, 5))\n\ndef drop_nan(df, tag):\n    return df[~df[tag].isna()].loc[:, tag]\n\nfig=plt.subplot(2,1,1)\nplt.title('Accuracy')\nplt.plot(drop_nan(df,tag='train_metric'),label='trn')\nplt.plot(drop_nan(df,tag='val_metric'),label='val')\nplt.legend(loc='best')\n\nfig=plt.subplot(2,1,2)\nplt.title('Loss')\nplt.plot(drop_nan(df,tag='train_loss'),label='trn')\nplt.plot(drop_nan(df,tag='val_loss'),label='val')\nplt.legend(loc='best')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:01:09.369052Z","iopub.execute_input":"2024-01-22T14:01:09.369383Z","iopub.status.idle":"2024-01-22T14:01:09.987289Z","shell.execute_reply.started":"2024-01-22T14:01:09.369355Z","shell.execute_reply":"2024-01-22T14:01:09.986356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# !unzip -q /kaggle/input/dogs-vs-cats/test1.zip -d /kaggle/working\n\ntst_ds=CatsDogsDataset('/kaggle/working/test1',val_transform,test=True)\ntst_loader=DataLoader(tst_ds,batch_size=25,shuffle=False,num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:03:05.014756Z","iopub.execute_input":"2024-01-22T14:03:05.015146Z","iopub.status.idle":"2024-01-22T14:03:05.030002Z","shell.execute_reply.started":"2024-01-22T14:03:05.015118Z","shell.execute_reply":"2024-01-22T14:03:05.028981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv('/kaggle/input/dogs-vs-cats/sampleSubmission.csv')\n\nmodel=trainer.model.model\n\nX=next(iter(tst_loader))\ny_pred=model(X)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:06:21.260629Z","iopub.execute_input":"2024-01-22T14:06:21.261094Z","iopub.status.idle":"2024-01-22T14:06:23.500260Z","shell.execute_reply.started":"2024-01-22T14:06:21.261055Z","shell.execute_reply":"2024-01-22T14:06:23.499031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:15:25.065477Z","iopub.execute_input":"2024-01-22T14:15:25.065837Z","iopub.status.idle":"2024-01-22T14:15:25.077624Z","shell.execute_reply.started":"2024-01-22T14:15:25.065810Z","shell.execute_reply":"2024-01-22T14:15:25.076613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ncols,nrows=6,4\nmean,std= torch.FloatTensor([0.485, 0.456, 0.406]),torch.FloatTensor([0.229, 0.224, 0.225])\nfig,axs=plt.subplots(nrows,ncols,figsize=(2*ncols,2*nrows))\nfor row in range(nrows):\n    for col in range(ncols):\n        idx=row*ncols+col\n        img=X[idx].permute(1,2,0)\n        axs[row,col].imshow(img*std[None,None,:]+mean[None,None,:])\n        axs[row,col].set_title('dog' if y_pred[idx]<0.5 else 'cat')\n        axs[row,col].axis('off')\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:13:51.450604Z","iopub.execute_input":"2024-01-22T14:13:51.451550Z","iopub.status.idle":"2024-01-22T14:13:54.069334Z","shell.execute_reply.started":"2024-01-22T14:13:51.451514Z","shell.execute_reply":"2024-01-22T14:13:54.068298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:24:28.855343Z","iopub.execute_input":"2024-01-22T14:24:28.856114Z","iopub.status.idle":"2024-01-22T14:24:28.860254Z","shell.execute_reply.started":"2024-01-22T14:24:28.856078Z","shell.execute_reply":"2024-01-22T14:24:28.859297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=0\nfrom tqdm.notebook import tqdm\nmodel=model.to(device)\nfor X in tqdm(tst_loader):\n    y_pred=model(X.to(device))\n    submission['label'].iloc[idx:idx+len(y_pred)]=(y_pred.detach().cpu()>0.5).type(torch.int16).flatten().numpy()\n    idx+=len(y_pred)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:25:30.615625Z","iopub.execute_input":"2024-01-22T14:25:30.616621Z","iopub.status.idle":"2024-01-22T14:26:06.628694Z","shell.execute_reply.started":"2024-01-22T14:25:30.616579Z","shell.execute_reply":"2024-01-22T14:26:06.627519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('Submission_valacc=098.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:26:10.924267Z","iopub.execute_input":"2024-01-22T14:26:10.925369Z","iopub.status.idle":"2024-01-22T14:26:10.957846Z","shell.execute_reply.started":"2024-01-22T14:26:10.925323Z","shell.execute_reply":"2024-01-22T14:26:10.956997Z"},"trusted":true},"execution_count":null,"outputs":[]}]}