{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Dogs vs Cats

\n \n
","metadata":{}},{"cell_type":"markdown","source":"

Introduction

\n

This notebook was created to learn PyTorch basics and to take a closer look to transfer learning technicks.
\nYou can use this work as a complete guide that can help to find answers for multiple questions \"How to...\":

\n\n","metadata":{}},{"cell_type":"markdown","source":"

1. Loading libriaries

","metadata":{}},{"cell_type":"code","source":"import os\nimport imp\nimport torch\nimport random\nimport torchvision\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom torch.utils.data import DataLoader\nfrom torchvision import transforms, models\n\n# ml lib\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:47:54.330070Z","iopub.execute_input":"2023-03-05T08:47:54.330439Z","iopub.status.idle":"2023-03-05T08:47:56.815826Z","shell.execute_reply.started":"2023-03-05T08:47:54.330346Z","shell.execute_reply":"2023-03-05T08:47:56.815050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global vars\n\nSEED = 127\nTRAIN_PATH = '../working/train'\nTEST_PATH = '../working/test1'\nTRAIN_BATCHSIZE = 32\nTEST_BATCHSIZE = 128\n\nEPOCHS_sfmmd = 7\nEPOCHS_densn121 = 7\nEPOCHS_resnet18 = 7\n\nDEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n\nREADY_MODELS_PATH = 'saved_models/'","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:47:56.817486Z","iopub.execute_input":"2023-03-05T08:47:56.817733Z","iopub.status.idle":"2023-03-05T08:47:56.888374Z","shell.execute_reply.started":"2023-03-05T08:47:56.817700Z","shell.execute_reply":"2023-03-05T08:47:56.887291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2. EDA

","metadata":{}},{"cell_type":"code","source":"# manual_seed\ntorch.manual_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:47:56.889697Z","iopub.execute_input":"2023-03-05T08:47:56.890099Z","iopub.status.idle":"2023-03-05T08:47:56.904131Z","shell.execute_reply.started":"2023-03-05T08:47:56.890063Z","shell.execute_reply":"2023-03-05T08:47:56.903475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.1. Data first look

","metadata":{}},{"cell_type":"code","source":"os.listdir('../input/dogs-vs-cats/')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:47:57.841136Z","iopub.execute_input":"2023-03-05T08:47:57.841468Z","iopub.status.idle":"2023-03-05T08:47:57.851537Z","shell.execute_reply.started":"2023-03-05T08:47:57.841435Z","shell.execute_reply":"2023-03-05T08:47:57.850788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -q \"/kaggle/input/dogs-vs-cats/test1.zip\"\n!unzip -q \"/kaggle/input/dogs-vs-cats/train.zip\"","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:47:59.424745Z","iopub.execute_input":"2023-03-05T08:47:59.425029Z","iopub.status.idle":"2023-03-05T08:48:16.861604Z","shell.execute_reply.started":"2023-03-05T08:47:59.424997Z","shell.execute_reply":"2023-03-05T08:48:16.860495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get labels\n\nlabels_list = []\ntrain_imgs = os.listdir('../working/train')\n\nfor f in train_imgs:\n \n # 0 for 'cat', 1 for 'dog'\n if f.split('.')[0] == 'cat':\n labels_list.append(0)\n\n elif f.split('.')[0] == 'dog':\n labels_list.append(1)\n\n else:\n # if no label \n labels_list.append(-1)\n\nprint(f'Images count = {len(train_imgs)}\\nLabels count = {len(labels_list)}')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:16.864284Z","iopub.execute_input":"2023-03-05T08:48:16.864619Z","iopub.status.idle":"2023-03-05T08:48:16.899962Z","shell.execute_reply.started":"2023-03-05T08:48:16.864568Z","shell.execute_reply":"2023-03-05T08:48:16.899042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.DataFrame(columns= ['image', 'label'])\ndf_train['image'] = train_imgs\ndf_train['label'] = labels_list\n\ndf_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:16.901321Z","iopub.execute_input":"2023-03-05T08:48:16.901751Z","iopub.status.idle":"2023-03-05T08:48:16.944002Z","shell.execute_reply.started":"2023-03-05T08:48:16.901712Z","shell.execute_reply":"2023-03-05T08:48:16.943092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['label'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:16.946094Z","iopub.execute_input":"2023-03-05T08:48:16.946435Z","iopub.status.idle":"2023-03-05T08:48:16.955657Z","shell.execute_reply.started":"2023-03-05T08:48:16.946399Z","shell.execute_reply":"2023-03-05T08:48:16.954736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize'] = (12,10)\nax = sns.countplot(data= df_train, x= 'label', palette= 'dark')\nax.set_title('Count of labels')\n","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:16.957071Z","iopub.execute_input":"2023-03-05T08:48:16.957424Z","iopub.status.idle":"2023-03-05T08:48:17.337790Z","shell.execute_reply.started":"2023-03-05T08:48:16.957388Z","shell.execute_reply":"2023-03-05T08:48:17.337090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shape_arr = []\n\nfor i in range(0, df_train.shape[0]):\n img_path = os.path.join(TRAIN_PATH, df_train['image'][i])\n\n with Image.open(img_path) as img:\n width, height = img.size\n shape_arr.append((width, height))\n\nnp.unique(shape_arr, axis= 0).__len__()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:17.338977Z","iopub.execute_input":"2023-03-05T08:48:17.339222Z","iopub.status.idle":"2023-03-05T08:48:19.710144Z","shell.execute_reply.started":"2023-03-05T08:48:17.339173Z","shell.execute_reply":"2023-03-05T08:48:19.705576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.max(shape_arr, axis= 0)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:19.713134Z","iopub.execute_input":"2023-03-05T08:48:19.713372Z","iopub.status.idle":"2023-03-05T08:48:19.733129Z","shell.execute_reply.started":"2023-03-05T08:48:19.713345Z","shell.execute_reply":"2023-03-05T08:48:19.732251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.min(shape_arr, axis= 0)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:19.736359Z","iopub.execute_input":"2023-03-05T08:48:19.736592Z","iopub.status.idle":"2023-03-05T08:48:19.757535Z","shell.execute_reply.started":"2023-03-05T08:48:19.736532Z","shell.execute_reply":"2023-03-05T08:48:19.756729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

NOTE: 8513 unique shapes of images, so reshape need.

","metadata":{}},{"cell_type":"code","source":"img_path = os.path.join(TRAIN_PATH, df_train['image'][1])\n\nwith Image.open(img_path) as img:\n plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:48:19.758757Z","iopub.execute_input":"2023-03-05T08:48:19.759972Z","iopub.status.idle":"2023-03-05T08:48:20.009877Z","shell.execute_reply.started":"2023-03-05T08:48:19.759936Z","shell.execute_reply":"2023-03-05T08:48:20.009151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.2. Mean and std to reshape and normalize

","metadata":{}},{"cell_type":"markdown","source":"## Mean (channels)","metadata":{}},{"cell_type":"code","source":"r_mean_arr = []\ng_mean_arr = []\nb_mean_arr = []\n\nfor i in range(0, df_train.shape[0]):\n img_path = os.path.join(TRAIN_PATH, df_train['image'][i])\n \n with Image.open(img_path) as img:\n img_np = np.array(img.getdata()).reshape(img.size[0], img.size[1], 3)\n\n r_mean, g_mean, b_mean = np.mean(img_np, axis= (0,1))\n\n r_mean_arr.append(r_mean)\n g_mean_arr.append(g_mean)\n b_mean_arr.append(b_mean)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T14:26:08.460846Z","iopub.execute_input":"2023-03-03T14:26:08.461202Z","iopub.status.idle":"2023-03-03T14:26:34.290659Z","shell.execute_reply.started":"2023-03-03T14:26:08.461167Z","shell.execute_reply":"2023-03-03T14:26:34.289400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"R_MEAN = np.mean(r_mean_arr) / 255\nG_MEAN = np.mean(g_mean_arr) / 255\nB_MEAN = np.mean(b_mean_arr) / 255","metadata":{"execution":{"iopub.status.busy":"2023-03-03T14:26:34.291624Z","iopub.status.idle":"2023-03-03T14:26:34.292100Z","shell.execute_reply.started":"2023-03-03T14:26:34.291864Z","shell.execute_reply":"2023-03-03T14:26:34.291887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Red ch mean = {R_MEAN}\\nGreen ch mean = {G_MEAN}\\nBlue ch mean = {B_MEAN}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-03T14:26:34.293886Z","iopub.status.idle":"2023-03-03T14:26:34.294619Z","shell.execute_reply.started":"2023-03-03T14:26:34.294374Z","shell.execute_reply":"2023-03-03T14:26:34.294399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Std (channels)","metadata":{}},{"cell_type":"code","source":"r_std_arr = []\ng_std_arr = []\nb_std_arr = []\n\nfor i in range(0, df_train.shape[0]):\n img_path = os.path.join(TRAIN_PATH, df_train['image'][i])\n \n with Image.open(img_path) as img:\n img_np = np.array(img.getdata()).reshape(img.size[0], img.size[1], 3)\n\n r_std, g_std, b_std = np.std(img_np, axis= (0,1))\n\n r_std_arr.append(r_std)\n g_std_arr.append(g_std)\n b_std_arr.append(b_std)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-03T14:26:34.295922Z","iopub.status.idle":"2023-03-03T14:26:34.296835Z","shell.execute_reply.started":"2023-03-03T14:26:34.296592Z","shell.execute_reply":"2023-03-03T14:26:34.296616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"R_STD = np.mean(r_std_arr) / 255\nG_STD = np.mean(g_std_arr) / 255\nB_STD = np.mean(b_std_arr) / 255","metadata":{"execution":{"iopub.status.busy":"2023-03-03T14:26:34.297915Z","iopub.status.idle":"2023-03-03T14:26:34.298799Z","shell.execute_reply.started":"2023-03-03T14:26:34.298568Z","shell.execute_reply":"2023-03-03T14:26:34.298592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Red ch std = {R_STD}\\nGreen ch std = {G_STD}\\nBlue ch std = {B_STD}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-03T14:26:34.299877Z","iopub.status.idle":"2023-03-03T14:26:34.300779Z","shell.execute_reply.started":"2023-03-03T14:26:34.300543Z","shell.execute_reply":"2023-03-03T14:26:34.300568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.3. Train/test split and samples view

","metadata":{}},{"cell_type":"code","source":"\nX_train, X_test, y_train, y_test = train_test_split(df_train['image'], df_train['label'], test_size= 0.15, random_state= SEED, stratify= df_train['label'])\n\nprint(f'Train size = {X_train.shape[0]}\\nTest size = {X_test.shape[0]}')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:49:12.247103Z","iopub.execute_input":"2023-03-05T08:49:12.247706Z","iopub.status.idle":"2023-03-05T08:49:12.265436Z","shell.execute_reply.started":"2023-03-05T08:49:12.247668Z","shell.execute_reply":"2023-03-05T08:49:12.264679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stratified check\n\nfig, axs = plt.subplots(1,2, figsize= (15,7))\n\nsns.countplot(x= y_train, ax= axs[0], palette= 'dark')\nsns.countplot(x= y_test, ax= axs[1], palette= 'bright')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:49:13.338077Z","iopub.execute_input":"2023-03-05T08:49:13.338665Z","iopub.status.idle":"2023-03-05T08:49:13.624611Z","shell.execute_reply.started":"2023-03-05T08:49:13.338627Z","shell.execute_reply":"2023-03-05T08:49:13.623892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforms","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose(\n [transforms.Resize((128, 128)), \n transforms.RandomHorizontalFlip(),\n transforms.ToTensor(), \n #transforms.Normalize([R_MEAN, G_MEAN, B_MEAN], [R_STD, G_STD, B_STD])])\n transforms.Normalize(mean= [0.488, 0.455, 0.417], std= [0.229, 0.225, 0.225])])","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:49:23.141006Z","iopub.execute_input":"2023-03-05T08:49:23.141292Z","iopub.status.idle":"2023-03-05T08:49:23.147425Z","shell.execute_reply.started":"2023-03-05T08:49:23.141259Z","shell.execute_reply":"2023-03-05T08:49:23.146699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_transforms = transforms.Compose(\n [transforms.Resize((128, 128)),\n transforms.ToTensor(), \n #transforms.Normalize([R_MEAN, G_MEAN, B_MEAN], [R_STD, G_STD, B_STD])])\n transforms.Normalize(mean= [0.488, 0.455, 0.417], std= [0.229, 0.225, 0.225])])","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:49:23.624760Z","iopub.execute_input":"2023-03-05T08:49:23.625028Z","iopub.status.idle":"2023-03-05T08:49:23.629888Z","shell.execute_reply.started":"2023-03-05T08:49:23.624997Z","shell.execute_reply":"2023-03-05T08:49:23.629012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating train/test custom datasets and loader","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2023-03-05T08:49:25.902811Z","iopub.execute_input":"2023-03-05T08:49:25.903103Z","iopub.status.idle":"2023-03-05T08:49:25.907336Z","shell.execute_reply.started":"2023-03-05T08:49:25.903064Z","shell.execute_reply":"2023-03-05T08:49:25.906474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DogsCatsData(Dataset):\n def __init__(self, x, y, transform=None, submission= False):\n self.x = x.reset_index()\n self.y = y.reset_index()\n self.submission = submission\n self.transform = transform\n \n def __len__(self):\n return self.x.shape[0]\n \n def load_image(self, path):\n prefix = TRAIN_PATH\n if self.submission:\n prefix = TEST_PATH\n #return np.array(Image.open(os.path.join(prefix, path)))\n return Image.open(os.path.join(prefix, path['image']))\n \n def __getitem__(self, index):\n image = self.load_image(self.x.iloc[index])\n label = self.y.iloc[index]['label']\n if self.transform:\n image = self.transform(image)\n if self.submission:\n image = np.array(image)\n sample = {'image': image, 'label': label}\n return sample","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:46:22.403731Z","iopub.execute_input":"2023-03-05T10:46:22.404005Z","iopub.status.idle":"2023-03-05T10:46:22.411474Z","shell.execute_reply.started":"2023-03-05T10:46:22.403974Z","shell.execute_reply":"2023-03-05T10:46:22.410667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = DogsCatsData(X_train, y_train, transform= train_transforms)\ntest_data = DogsCatsData(X_test, y_test, transform= test_transforms)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:18:58.073075Z","iopub.execute_input":"2023-03-05T10:18:58.073783Z","iopub.status.idle":"2023-03-05T10:18:58.085692Z","shell.execute_reply.started":"2023-03-05T10:18:58.073746Z","shell.execute_reply":"2023-03-05T10:18:58.084924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainloader = torch.utils.data.DataLoader(train_data,batch_size=32, num_workers= 2)\ntestloader = torch.utils.data.DataLoader(test_data, batch_size=32, num_workers= 2)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:18:59.951876Z","iopub.execute_input":"2023-03-05T10:18:59.952464Z","iopub.status.idle":"2023-03-05T10:18:59.957158Z","shell.execute_reply.started":"2023-03-05T10:18:59.952425Z","shell.execute_reply":"2023-03-05T10:18:59.956272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i_batch, sample_batched in enumerate(trainloader):\n print(i_batch, sample_batched['image'].size(),\n sample_batched['label'].size())\n \n if i_batch > 4:\n break\n","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:13:28.487513Z","iopub.execute_input":"2023-03-05T09:13:28.487786Z","iopub.status.idle":"2023-03-05T09:13:29.435045Z","shell.execute_reply.started":"2023-03-05T09:13:28.487757Z","shell.execute_reply":"2023-03-05T09:13:29.434158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

3. Model selection

","metadata":{}},{"cell_type":"markdown","source":"

3.1. Implementing self-made CNN model

","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\n\nclass ConvNet(nn.Module):\n def __init__(self):\n super(ConvNet, self).__init__()\n\n # layer1\n self.cv1 = nn.Conv2d(in_channels= 3, out_channels= 32, kernel_size= 3)\n self.batch_norm1 = nn.BatchNorm2d(num_features= 32)\n self.relu1 = nn.ReLU()\n self.max_pool1 = nn.MaxPool2d(2)\n\n # layer2\n self.cv2 = nn.Conv2d(in_channels= 32, out_channels= 64, kernel_size= 3)\n self.batch_norm2 = nn.BatchNorm2d(num_features= 64)\n self.relu2 = nn.ReLU()\n\n\n # layer3\n self.cv3 = nn.Conv2d(in_channels= 64, out_channels= 80, kernel_size= 3)\n self.batch_norm3 = nn.BatchNorm2d(num_features= 80)\n self.relu3 = nn.ReLU()\n self.max_pool3 = nn.MaxPool2d(2)\n\n # fc1\n self.fc1 = nn.Linear(67280, 2000)\n self.relu_fc1 = nn.ReLU()\n\n # fc1\n self.fc2 = nn.Linear(2000, 500)\n self.relu_fc2 = nn.ReLU()\n\n # fc1\n self.fc3 = nn.Linear(500, 2)\n #self.sfmx3 = nn.LogSoftmax()\n\n\n def forward(self, x):\n # l1\n x = self.max_pool1(self.relu1(self.batch_norm1(self.cv1(x))))\n\n # l2\n x = self.relu2(self.batch_norm2(self.cv2(x)))\n\n # l3\n x = self.max_pool3(self.relu3(self.batch_norm3(self.cv3(x))))\n\n x = x.view(x.size(0), -1)\n\n\n # fc 1,2,3\n x = self.relu_fc1(self.fc1(x))\n x = self.relu_fc2(self.fc2(x))\n\n x = self.fc3(x)\n #x = self.sfmx3(x)\n \n return x","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:27:40.031882Z","iopub.execute_input":"2023-03-05T09:27:40.032229Z","iopub.status.idle":"2023-03-05T09:27:40.043975Z","shell.execute_reply.started":"2023-03-05T09:27:40.032188Z","shell.execute_reply":"2023-03-05T09:27:40.043197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tools to visualize model\n!pip install graphviz\n!pip install torchviz","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-05T09:26:38.626354Z","iopub.execute_input":"2023-03-05T09:26:38.626623Z","iopub.status.idle":"2023-03-05T09:26:54.053063Z","shell.execute_reply.started":"2023-03-05T09:26:38.626595Z","shell.execute_reply":"2023-03-05T09:26:54.052118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = ConvNet()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:38:51.089574Z","iopub.execute_input":"2023-03-05T09:38:51.090193Z","iopub.status.idle":"2023-03-05T09:38:52.328458Z","shell.execute_reply.started":"2023-03-05T09:38:51.090157Z","shell.execute_reply":"2023-03-05T09:38:52.327633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchviz import make_dot\nbatch = next(iter(trainloader))\nyhat = net(batch['image']) # Give dummy batch to forward().\nmake_dot(y, params=dict(list(net.named_parameters())))","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:27:43.332760Z","iopub.execute_input":"2023-03-05T09:27:43.333070Z","iopub.status.idle":"2023-03-05T09:27:45.218548Z","shell.execute_reply.started":"2023-03-05T09:27:43.332997Z","shell.execute_reply":"2023-03-05T09:27:45.214754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train function","metadata":{}},{"cell_type":"code","source":"def train_cycle(model, optimizer, criterion, p_iter, n_epochs):\n model.train()\n model.to(DEVICE)\n itr = 1\n total_loss = 0\n loss_list = []\n acc_list = []\n for epoch in range(n_epochs):\n for batch_no, data in enumerate(trainloader, 0):\n\n samples, labels = data['image'], data['label']\n samples = samples.to(DEVICE)\n labels = labels.to(DEVICE)\n \n optimizer.zero_grad()\n\n output = model(samples)\n loss = criterion(output, labels)\n loss.backward()\n optimizer.step()\n total_loss += loss.item()\n\n if itr%p_iter == 0:\n pred = torch.argmax(output, dim=1)\n correct = pred.eq(labels)\n acc = torch.mean(correct.float())\n acc = acc.to('cpu')\n print('[Epoch {}/{}] Iteration {} -> Train Loss: {:.4f}, Accuracy: {:.3f}'.format(epoch+1, n_epochs, itr, total_loss/p_iter, acc))\n loss_list.append(total_loss/p_iter)\n acc_list.append(acc)\n total_loss = 0\n\n itr += 1\n\n plt.plot(loss_list[1:], label='loss')\n plt.plot(acc_list[1:], label='accuracy')\n plt.legend()\n plt.title('training loss and accuracy')\n plt.show()\n print('Finished Training')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:38:57.051855Z","iopub.execute_input":"2023-03-05T09:38:57.052177Z","iopub.status.idle":"2023-03-05T09:38:57.066078Z","shell.execute_reply.started":"2023-03-05T09:38:57.052146Z","shell.execute_reply":"2023-03-05T09:38:57.065364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(net.parameters(), lr=0.001)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:38:57.488430Z","iopub.execute_input":"2023-03-05T09:38:57.488687Z","iopub.status.idle":"2023-03-05T09:38:57.494011Z","shell.execute_reply.started":"2023-03-05T09:38:57.488659Z","shell.execute_reply":"2023-03-05T09:38:57.493129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Self-made model train\n\ntrain_cycle(net, optimizer, criterion, 200, n_epochs= EPOCHS_sfmmd)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:38:58.861133Z","iopub.execute_input":"2023-03-05T09:38:58.861919Z","iopub.status.idle":"2023-03-05T09:48:14.099500Z","shell.execute_reply.started":"2023-03-05T09:38:58.861879Z","shell.execute_reply":"2023-03-05T09:48:14.098639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

3.2. Transfer learning attempts

","metadata":{}},{"cell_type":"markdown","source":"## Densnet121(pretrained)","metadata":{}},{"cell_type":"code","source":"#torchvision.models\n\ndensn121 = models.densenet121(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:48:45.255547Z","iopub.execute_input":"2023-03-05T09:48:45.255813Z","iopub.status.idle":"2023-03-05T09:48:47.453240Z","shell.execute_reply.started":"2023-03-05T09:48:45.255783Z","shell.execute_reply":"2023-03-05T09:48:47.452481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(densn121.parameters(), lr=0.007)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:48:49.899323Z","iopub.execute_input":"2023-03-05T09:48:49.899598Z","iopub.status.idle":"2023-03-05T09:48:49.905651Z","shell.execute_reply.started":"2023-03-05T09:48:49.899568Z","shell.execute_reply":"2023-03-05T09:48:49.904965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_ftrs = densn121.classifier.in_features\ndensn121.classifier = nn.Sequential(\n nn.Linear(num_ftrs, 500),\n nn.Linear(500, 2)\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:48:52.524469Z","iopub.execute_input":"2023-03-05T09:48:52.524747Z","iopub.status.idle":"2023-03-05T09:48:52.536297Z","shell.execute_reply.started":"2023-03-05T09:48:52.524717Z","shell.execute_reply":"2023-03-05T09:48:52.535420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cycle(densn121, optimizer= optimizer, criterion= criterion, p_iter= 200, n_epochs= EPOCHS_densn121)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T09:49:05.637815Z","iopub.execute_input":"2023-03-05T09:49:05.638645Z","iopub.status.idle":"2023-03-05T10:02:20.893079Z","shell.execute_reply.started":"2023-03-05T09:49:05.638606Z","shell.execute_reply":"2023-03-05T10:02:20.892292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet18","metadata":{}},{"cell_type":"code","source":"resnet18 = models.resnet18(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:02:20.895677Z","iopub.execute_input":"2023-03-05T10:02:20.896196Z","iopub.status.idle":"2023-03-05T10:02:23.498893Z","shell.execute_reply.started":"2023-03-05T10:02:20.896157Z","shell.execute_reply":"2023-03-05T10:02:23.498133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(resnet18.parameters(), lr=0.009)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:02:23.503263Z","iopub.execute_input":"2023-03-05T10:02:23.503706Z","iopub.status.idle":"2023-03-05T10:02:23.509081Z","shell.execute_reply.started":"2023-03-05T10:02:23.503665Z","shell.execute_reply":"2023-03-05T10:02:23.508438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_ftrs = resnet18.fc.in_features\nresnet18.fc = nn.Sequential(\n nn.Linear(num_ftrs, 500),\n nn.Linear(500, 2)\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:02:23.511101Z","iopub.execute_input":"2023-03-05T10:02:23.512006Z","iopub.status.idle":"2023-03-05T10:02:23.525465Z","shell.execute_reply.started":"2023-03-05T10:02:23.511971Z","shell.execute_reply":"2023-03-05T10:02:23.524863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cycle(resnet18, optimizer= optimizer, criterion= criterion, p_iter= 200, n_epochs= EPOCHS_resnet18)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:02:23.526533Z","iopub.execute_input":"2023-03-05T10:02:23.527017Z","iopub.status.idle":"2023-03-05T10:11:37.323185Z","shell.execute_reply.started":"2023-03-05T10:02:23.526980Z","shell.execute_reply":"2023-03-05T10:11:37.322387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4. Predictions and conclusions

","metadata":{}},{"cell_type":"markdown","source":"

4.1. Comparative tests

","metadata":{}},{"cell_type":"code","source":"def test_model(model, test_data, criterion):\n\n model.eval()\n\n acc_list = []\n loss_list = []\n val_loss = 0\n with torch.no_grad():\n for batch_no, data in tqdm(enumerate(test_data, 0)):\n samples, labels = data['image'], data['label']\n samples = samples.to(DEVICE)\n labels = labels.to(DEVICE)\n \n output = model(samples)\n loss = criterion(output, labels)\n \n val_loss += loss.item()\n\n pred = torch.argmax(output, dim=1)\n correct = pred.eq(labels)\n acc = torch.mean(correct.float())\n acc = acc.to('cpu')\n acc_list.append(acc)\n loss = loss.to('cpu')\n loss_list.append(loss)\n\n print(f'Mean acc = {np.mean(acc_list): .3f}. Mean loss = {np.mean(loss_list): .3f}')\n\n ","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:22:38.751118Z","iopub.execute_input":"2023-03-05T10:22:38.751737Z","iopub.status.idle":"2023-03-05T10:22:38.761055Z","shell.execute_reply.started":"2023-03-05T10:22:38.751700Z","shell.execute_reply":"2023-03-05T10:22:38.760238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:22:39.154870Z","iopub.execute_input":"2023-03-05T10:22:39.155148Z","iopub.status.idle":"2023-03-05T10:22:39.159087Z","shell.execute_reply.started":"2023-03-05T10:22:39.155116Z","shell.execute_reply":"2023-03-05T10:22:39.158263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Self-made ConvNet:')\ntest_model(net, testloader, criterion)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:22:39.571852Z","iopub.execute_input":"2023-03-05T10:22:39.572113Z","iopub.status.idle":"2023-03-05T10:22:52.324532Z","shell.execute_reply.started":"2023-03-05T10:22:39.572085Z","shell.execute_reply":"2023-03-05T10:22:52.322190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('ResNet18:')\ntest_model(resnet18, testloader, criterion)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:22:59.759232Z","iopub.execute_input":"2023-03-05T10:22:59.759641Z","iopub.status.idle":"2023-03-05T10:23:13.001018Z","shell.execute_reply.started":"2023-03-05T10:22:59.759602Z","shell.execute_reply":"2023-03-05T10:23:12.999932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('DensNet121:')\ntest_model(densn121, testloader, criterion)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:23:13.003687Z","iopub.execute_input":"2023-03-05T10:23:13.003997Z","iopub.status.idle":"2023-03-05T10:23:26.982561Z","shell.execute_reply.started":"2023-03-05T10:23:13.003955Z","shell.execute_reply":"2023-03-05T10:23:26.980463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.2. Save models

","metadata":{}},{"cell_type":"code","source":"os.makedirs(os.path.join('../working/', READY_MODELS_PATH))","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:25:59.139609Z","iopub.execute_input":"2023-03-05T10:25:59.139914Z","iopub.status.idle":"2023-03-05T10:25:59.145273Z","shell.execute_reply.started":"2023-03-05T10:25:59.139883Z","shell.execute_reply":"2023-03-05T10:25:59.144429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntorch.save(net.state_dict(), os.path.join(READY_MODELS_PATH, 'smConvNet.pt'))\n\n\n# TO LOAD:\n#model = TheModelClass(*args, **kwargs)\n#model.load_state_dict(torch.load(PATH))\n#model.eval()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:26:00.573568Z","iopub.execute_input":"2023-03-05T10:26:00.574305Z","iopub.status.idle":"2023-03-05T10:26:02.127027Z","shell.execute_reply.started":"2023-03-05T10:26:00.574269Z","shell.execute_reply":"2023-03-05T10:26:02.126256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(densn121.state_dict(), os.path.join(READY_MODELS_PATH, 'densn121.pt'))\n","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:26:07.511808Z","iopub.execute_input":"2023-03-05T10:26:07.512071Z","iopub.status.idle":"2023-03-05T10:26:07.622653Z","shell.execute_reply.started":"2023-03-05T10:26:07.512042Z","shell.execute_reply":"2023-03-05T10:26:07.621886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(resnet18.state_dict(), os.path.join(READY_MODELS_PATH, 'resnet18.pt'))","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:26:08.218801Z","iopub.execute_input":"2023-03-05T10:26:08.219070Z","iopub.status.idle":"2023-03-05T10:26:08.311017Z","shell.execute_reply.started":"2023-03-05T10:26:08.219040Z","shell.execute_reply":"2023-03-05T10:26:08.310256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.3. Load models

","metadata":{}},{"cell_type":"code","source":"cnn_m = ConvNet()\ncnn_m.load_state_dict(torch.load(os.path.join(READY_MODELS_PATH, 'smConvNet.pt')))\ncnn_m","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:32:32.137235Z","iopub.execute_input":"2023-03-05T10:32:32.137507Z","iopub.status.idle":"2023-03-05T10:32:34.028837Z","shell.execute_reply.started":"2023-03-05T10:32:32.137476Z","shell.execute_reply":"2023-03-05T10:32:34.028089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet18_loaded = models.resnet18()\n\nnum_ftrs = resnet18_loaded.fc.in_features\nresnet18_loaded.fc = nn.Sequential(\n nn.Linear(num_ftrs, 500),\n nn.Linear(500, 2)\n)\n\nresnet18_loaded.load_state_dict(torch.load(os.path.join(READY_MODELS_PATH, 'resnet18.pt')))","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:32:37.566319Z","iopub.execute_input":"2023-03-05T10:32:37.566588Z","iopub.status.idle":"2023-03-05T10:32:37.799679Z","shell.execute_reply.started":"2023-03-05T10:32:37.566560Z","shell.execute_reply":"2023-03-05T10:32:37.798616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"densn121_loaded = models.densenet121()\n\nnum_ftrs = densn121_loaded.classifier.in_features\ndensn121_loaded.classifier = nn.Sequential(\n nn.Linear(num_ftrs, 500),\n nn.Linear(500, 2)\n)\n\ndensn121_loaded.load_state_dict(torch.load(os.path.join(READY_MODELS_PATH, 'densn121.pt')))","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:32:39.138207Z","iopub.execute_input":"2023-03-05T10:32:39.138512Z","iopub.status.idle":"2023-03-05T10:32:39.464790Z","shell.execute_reply.started":"2023-03-05T10:32:39.138478Z","shell.execute_reply":"2023-03-05T10:32:39.463727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_list = []\nmodels_list.append(cnn_m)\nmodels_list.append(resnet18_loaded)\nmodels_list.append(densn121_loaded)\n\nfor i in models_list:\n print(type(i))","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:32:40.530610Z","iopub.execute_input":"2023-03-05T10:32:40.531164Z","iopub.status.idle":"2023-03-05T10:32:40.536862Z","shell.execute_reply.started":"2023-03-05T10:32:40.531127Z","shell.execute_reply":"2023-03-05T10:32:40.536139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.4. Predictions

","metadata":{}},{"cell_type":"code","source":"# Load test set for submission\n\ntest_imgs_f = os.listdir(TEST_PATH)\nlabels_list_fake = [0] * len(test_imgs_f)\ntest_ids = []\n\n\n\nfor file_name in test_imgs_f:\n test_ids.append(file_name.split('.')[0])\n\n\nprint(f'Images count = {len(test_ids)}\\nLabels count = {len(labels_list_fake)}')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:47:35.831735Z","iopub.execute_input":"2023-03-05T10:47:35.832028Z","iopub.status.idle":"2023-03-05T10:47:35.852131Z","shell.execute_reply.started":"2023-03-05T10:47:35.831996Z","shell.execute_reply":"2023-03-05T10:47:35.851194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_f = pd.DataFrame(columns= ['image', 'label'])\ndf_test_f['image'] = test_imgs_f\ndf_test_f['label'] = labels_list_fake","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:47:36.132072Z","iopub.execute_input":"2023-03-05T10:47:36.132647Z","iopub.status.idle":"2023-03-05T10:47:36.149533Z","shell.execute_reply.started":"2023-03-05T10:47:36.132608Z","shell.execute_reply":"2023-03-05T10:47:36.148650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g_test_data = DogsCatsData(df_test_f['image'], df_test_f['label'], transform= test_transforms, submission= True)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:47:36.826059Z","iopub.execute_input":"2023-03-05T10:47:36.826647Z","iopub.status.idle":"2023-03-05T10:47:36.835013Z","shell.execute_reply.started":"2023-03-05T10:47:36.826608Z","shell.execute_reply":"2023-03-05T10:47:36.834048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g_testloader = torch.utils.data.DataLoader(g_test_data,batch_size=TEST_BATCHSIZE, num_workers= 2)","metadata":{"execution":{"iopub.status.busy":"2023-03-05T10:47:37.498525Z","iopub.execute_input":"2023-03-05T10:47:37.498790Z","iopub.status.idle":"2023-03-05T10:47:37.503160Z","shell.execute_reply.started":"2023-03-05T10:47:37.498759Z","shell.execute_reply":"2023-03-05T10:47:37.502461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in models_list:\n\n preds_final = torch.Tensor()\n preds_final = preds_final.to(DEVICE)\n \n predictions_frame = pd.DataFrame(columns= ['ids', 'preds'])\n predictions_frame['ids'] = test_ids\n\n i.eval()\n i.to(DEVICE)\n with torch.no_grad():\n for batch_no, data in tqdm(enumerate(g_testloader, 0)):\n samples, labels = data['image'], data['label']\n samples = samples.to(DEVICE)\n labels = labels.to(DEVICE)\n \n output = i(samples)\n \n pred = torch.argmax(output, dim=1)\n preds_final = torch.cat((preds_final, pred), 0)\n #print(len(preds_final))\n \n preds_final = preds_final.to('cpu')\n predictions_frame['preds'] = preds_final\n \n name = str(i)\n name = name[:7].strip()\n predictions_frame.to_csv(f'{name}.csv', header= False, index= False)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-05T11:00:58.332646Z","iopub.execute_input":"2023-03-05T11:00:58.332923Z","iopub.status.idle":"2023-03-05T11:03:04.755216Z","shell.execute_reply.started":"2023-03-05T11:00:58.332891Z","shell.execute_reply":"2023-03-05T11:03:04.754387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Records check\n\npreds_fchech = pd.read_csv('ConvNet.csv', header= None)\nprint(f'Self-made CNN preds ---{preds_fchech[1].unique()}')\n\npreds_fchech = pd.read_csv('DenseNe.csv', header= None)\nprint(f'DensNet121 preds ---{preds_fchech[1].unique()}')\n\npreds_fchech = pd.read_csv('ResNet(.csv', header= None)\nprint(f'Resnet18 preds ---{preds_fchech[1].unique()}')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T11:03:10.346834Z","iopub.execute_input":"2023-03-05T11:03:10.347131Z","iopub.status.idle":"2023-03-05T11:03:10.375430Z","shell.execute_reply.started":"2023-03-05T11:03:10.347101Z","shell.execute_reply":"2023-03-05T11:03:10.374676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clear output to save nb\nimport shutil\nshutil.rmtree('../working/train')\nshutil.rmtree('../working/test1')\nshutil.rmtree('../working/saved_models')","metadata":{"execution":{"iopub.status.busy":"2023-03-05T11:11:33.428117Z","iopub.execute_input":"2023-03-05T11:11:33.428420Z","iopub.status.idle":"2023-03-05T11:11:34.859738Z","shell.execute_reply.started":"2023-03-05T11:11:33.428381Z","shell.execute_reply":"2023-03-05T11:11:34.858916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Conclusion

\n

As we can see transfer learning gets better resuls in practice.
\nSelf-made CNN also got not bad results even it was small.
\nUpvote! If it was usefull.

\n","metadata":{}}]}