{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"},{"sourceId":65911,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":54965}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from PIL import Image, ImageFont, ImageDraw \nimport torch \nfrom torch import nn\nfrom torch.utils.data import DataLoader , Dataset\nimport numpy as np\nimport pandas as pd \nimport math\nimport matplotlib.pyplot as plt\nimport glob \nimport torch.nn.functional as F\nimport os, sys, gc, time, warnings, pickle, psutil, random\nos.mkdir(\"/kaggle/working/model_checkpoints\")\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice,torch.__version__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-16T08:23:49.078819Z","iopub.execute_input":"2024-06-16T08:23:49.079633Z","iopub.status.idle":"2024-06-16T08:23:49.088604Z","shell.execute_reply.started":"2024-06-16T08:23:49.079595Z","shell.execute_reply":"2024-06-16T08:23:49.087653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:50.133947Z","iopub.execute_input":"2024-06-16T08:23:50.134317Z","iopub.status.idle":"2024-06-16T08:23:50.23024Z","shell.execute_reply.started":"2024-06-16T08:23:50.13429Z","shell.execute_reply":"2024-06-16T08:23:50.229251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# functions\n","metadata":{}},{"cell_type":"code","source":"model_dir = \"/kaggle/input/gan-confidence-estimation/pytorch/checkpoint260/1\"","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:51.562745Z","iopub.execute_input":"2024-06-16T08:23:51.563324Z","iopub.status.idle":"2024-06-16T08:23:51.56779Z","shell.execute_reply.started":"2024-06-16T08:23:51.563296Z","shell.execute_reply":"2024-06-16T08:23:51.566861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multiclass_calibration_curve(probs, labels, bins=20):\n    step_size = 1.0 / bins\n    n_classes = probs.shape[1]\n    labels_ohe = np.eye(n_classes)[labels.astype(np.int64)]\n \n    midpoints = []\n    mean_confidences = []\n    accuracies = []\n    ece = []\n    for i in range(bins):\n        beg = i * step_size\n        end = (i + 1) * step_size\n \n        bin_mask = (probs >= beg) & (probs < end)\n        bin_cnt = bin_mask.astype(np.float32).sum()\n        bin_confs = probs[bin_mask]\n        bin_acc = labels_ohe[bin_mask].sum() / bin_cnt\n \n        midpoints.append((beg+end)/2.)\n        mean_confidences.append(np.mean(bin_confs))\n        accuracies.append(bin_acc)\n        ece.append(bin_cnt*np.abs(mean_confidences[-1] - accuracies[-1]))\n    return midpoints, accuracies, mean_confidences, ece\n\ndef plot_multiclass_calibration_curve(probs, labels, bins=10, title=None):\n    title = 'Reliability Diagram' if title is None else title\n    midpoints, accuracies, mean_confidences, ece = multiclass_calibration_curve(probs, labels, bins=bins)\n    plt.bar(midpoints, accuracies, width=1.0/float(bins), align='center', lw=1, ec='#000000', fc='#2233aa', alpha=1, label='Model', zorder=0)\n    plt.scatter(midpoints, accuracies, lw=2, ec='black', fc=\"#ffffff\", zorder=2)\n    plt.plot(np.linspace(0, 1.0, 20), np.linspace(0, 1.0, 20), '--', lw=2, alpha=.7, color='gray', label='Perfectly calibrated', zorder=1)\n    plt.xlim(0.0, 1.0)\n    plt.ylim(0.0, 1.0)\n    plt.xlabel('\\nconfidence')\n    plt.ylabel('accuracy\\n')\n    plt.title(title+'\\n')\n    plt.xticks(midpoints, rotation=-45)\n    plt.legend(loc='upper left')\n    ece = np.array(ece).sum()/100\n    return midpoints, accuracies, mean_confidences, ece","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:51.77978Z","iopub.execute_input":"2024-06-16T08:23:51.780177Z","iopub.status.idle":"2024-06-16T08:23:51.79331Z","shell.execute_reply.started":"2024-06-16T08:23:51.780148Z","shell.execute_reply":"2024-06-16T08:23:51.792383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_grid(images, nrows=5, ncols=5, figsize=(10, 10)):\n    \"\"\"\n    Plots a grid of images from a PyTorch tensor.\n\n    Args:\n        images (torch.Tensor): A tensor of shape (batch_size, channels, height, width) containing the images.\n        nrows (int, optional): Number of rows in the grid. Defaults to 5.\n        ncols (int, optional): Number of columns in the grid. Defaults to 5.\n        figsize (tuple, optional): Size of the figure in inches. Defaults to (10, 10).\n    \"\"\"\n\n    num_images = images.shape[0]\n    if num_images != nrows * ncols:\n        raise ValueError(f\"Number of images ({num_images}) must be a multiple of grid size ({nrows}x{ncols})\")\n\n    # Ensure images are in the range [0, 1] for proper display\n    images = images.clamp(0, 1)\n\n    # Create a new figure with specific size\n    plt.figure(figsize=figsize)\n\n    # Loop through images and plot them on a grid\n    for i in range(num_images):\n        plt.subplot(nrows, ncols, i + 1)  # Start subplot indexing from 1\n        plt.imshow(images[i].permute(1, 2, 0).cpu().numpy() , cmap = \"gray\")  # Permute and move to CPU\n        plt.axis('off')  # Turn off axes for cleaner visualization\n\n    # Tight layout to prevent overlapping labels and titles\n    plt.tight_layout()\n    plt.show()\n    \ndef plot_confidence_score(images, nrows=5, ncols=5, figsize=(10, 10)):\n    \"\"\"\n    Plots a grid of images from a PyTorch tensor.\n\n    Args:\n        images (torch.Tensor): A tensor of shape (batch_size, channels, height, width) containing the images.\n        nrows (int, optional): Number of rows in the grid. Defaults to 5.\n        ncols (int, optional): Number of columns in the grid. Defaults to 5.\n        figsize (tuple, optional): Size of the figure in inches. Defaults to (10, 10).\n    \"\"\"\n\n    num_images = images.shape[0]\n    if num_images != nrows * ncols:\n        raise ValueError(f\"Number of images ({num_images}) must be a multiple of grid size ({nrows}x{ncols})\")\n\n    # Ensure images are in the range [0, 1] for proper display\n    images = images.clamp(0, 1)\n\n    # Create a new figure with specific size\n    plt.figure(figsize=figsize)\n\n    # Loop through images and plot them on a grid\n    for i in range(num_images):\n        plt.subplot(nrows, ncols, i + 1)  # Start subplot indexing from 1\n        plt.imshow(images[i].permute(1, 2, 0).cpu().numpy() , cmap = \"plasma\")  # Permute and move to CPU\n        plt.colorbar() \n        plt.axis('off')  # Turn off axes for cleaner visualization\n\n    # Tight layout to prevent overlapping labels and titles\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:52.037008Z","iopub.execute_input":"2024-06-16T08:23:52.037841Z","iopub.status.idle":"2024-06-16T08:23:52.049018Z","shell.execute_reply.started":"2024-06-16T08:23:52.037809Z","shell.execute_reply":"2024-06-16T08:23:52.048004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint(gen, dis, gen_opt, dis_opt, epoch, checkpoint_dir):\n    checkpoint = {\n        'gen_state_dict': gen.state_dict(),\n        'dis_state_dict': dis.state_dict(),\n        'gen_opt_state_dict': gen_opt.state_dict(),\n        'dis_opt_state_dict': dis_opt.state_dict(),\n        'epoch': epoch\n    }\n    torch.save(checkpoint, os.path.join(checkpoint_dir, f'checkpoint_{epoch}.pth'))\n\n# Function to load checkpoints if available\ndef load_checkpoint(gen, dis, gen_opt, dis_opt , checkpoint_dir = model_dir):\n    checkpoints = [f for f in os.listdir(checkpoint_dir) if f.startswith('checkpoint_')]\n    if not checkpoints:\n        return 0\n    latest_checkpoint = max(checkpoints, key=lambda x: int(x.split('_')[1].split('.')[0]))\n    checkpoint_path = os.path.join(checkpoint_dir, latest_checkpoint)\n    checkpoint = torch.load(checkpoint_path)\n    gen.load_state_dict(checkpoint['gen_state_dict'])\n    dis.load_state_dict(checkpoint['dis_state_dict'])\n    gen_opt.load_state_dict(checkpoint['gen_opt_state_dict'])\n    dis_opt.load_state_dict(checkpoint['dis_opt_state_dict'])\n    return checkpoint['epoch'] + 1","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:52.247121Z","iopub.execute_input":"2024-06-16T08:23:52.247477Z","iopub.status.idle":"2024-06-16T08:23:52.256352Z","shell.execute_reply.started":"2024-06-16T08:23:52.247448Z","shell.execute_reply":"2024-06-16T08:23:52.255398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataloader\n","metadata":{}},{"cell_type":"code","source":"# Training dataset class with normalization\nclass Build_Data(Dataset):\n    def __init__(self):\n        self.df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\n        self.x = self.df.iloc[:, 1:].values\n        self.len = self.x.shape[0]\n        \n    def __getitem__(self, index):\n        img = self.x[index].reshape(1,28, 28)\n        img = torch.tensor(img, dtype=torch.float32)\n        img = torch.log(img + 1) / math.log(256)  # Normalization\n        return img\n    \n    def __len__(self):\n        return self.len\n\n# Test dataset class with normalization\nclass Test_Data(Dataset):\n    def __init__(self):\n        self.df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n        self.x = self.df.values\n        self.len = self.x.shape[0]\n        \n    def __getitem__(self, index):\n        img = self.x[index].reshape(1,28, 28)\n        img = torch.tensor(img, dtype=torch.float32)\n        img = torch.log(img + 1) / math.log(256)  # Normalization\n        return img\n    \n    def __len__(self):\n        return self.len\n\n# Create train and test datasets\ntrain_dataset = Build_Data()\ntest_dataset = Test_Data()\n\n# Create data loaders\ntrain_data_loader = DataLoader(train_dataset, batch_size=1000, shuffle=True)\ntest_data_loader = DataLoader(test_dataset, batch_size=1000, shuffle=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:53.360279Z","iopub.execute_input":"2024-06-16T08:23:53.360938Z","iopub.status.idle":"2024-06-16T08:23:58.919529Z","shell.execute_reply.started":"2024-06-16T08:23:53.360906Z","shell.execute_reply":"2024-06-16T08:23:58.918735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Displaying one batch from train data loader\nwith torch.no_grad():\n    for batch in train_data_loader:\n        obj1 = batch[0]\n        break\n\nplt.imshow(obj1[0], cmap=\"gray\")\nplt.show()\n\n# Displaying one batch from test data loader\nwith torch.no_grad():\n    for batch in test_data_loader:\n        test_obj1 = batch[0]\n        print(test_obj1.shape)\n        break\n\nplt.imshow(test_obj1[0], cmap=\"gray\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:58.921189Z","iopub.execute_input":"2024-06-16T08:23:58.921474Z","iopub.status.idle":"2024-06-16T08:23:59.534274Z","shell.execute_reply.started":"2024-06-16T08:23:58.92145Z","shell.execute_reply":"2024-06-16T08:23:59.533422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading dataset","metadata":{}},{"cell_type":"code","source":"# number_of_samples = 10000\n# chunksize = number_of_samples\n# r = 0\n# for chunk in pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\",chunksize = chunksize):\n#     r += 1\n#     if r > 1:\n#         break\n#     df = chunk","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:23:59.53556Z","iopub.execute_input":"2024-06-16T08:23:59.535923Z","iopub.status.idle":"2024-06-16T08:23:59.540371Z","shell.execute_reply.started":"2024-06-16T08:23:59.535892Z","shell.execute_reply":"2024-06-16T08:23:59.539517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_fonts = df.iloc[0:number_of_samples,1:786]\n# all_fonts = np.array(all_fonts)\n# all_fonts = torch.tensor(all_fonts)","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:58:48.200183Z","iopub.execute_input":"2024-06-15T15:58:48.200931Z","iopub.status.idle":"2024-06-15T15:58:48.210129Z","shell.execute_reply.started":"2024-06-15T15:58:48.200875Z","shell.execute_reply":"2024-06-15T15:58:48.209403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split_amount = all_fonts.shape[0]//4\n# train_x = all_fonts[0:3*split_amount]\n# test_x = all_fonts[3*split_amount:-1]","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:58:48.211169Z","iopub.execute_input":"2024-06-15T15:58:48.211448Z","iopub.status.idle":"2024-06-15T15:58:48.219644Z","shell.execute_reply.started":"2024-06-15T15:58:48.211425Z","shell.execute_reply":"2024-06-15T15:58:48.218757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_x.shape,test_x.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.107387Z","iopub.execute_input":"2024-06-15T15:43:26.10772Z","iopub.status.idle":"2024-06-15T15:43:26.115526Z","shell.execute_reply.started":"2024-06-15T15:43:26.107686Z","shell.execute_reply":"2024-06-15T15:43:26.114683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import warnings\n# warnings.simplefilter('ignore')\n# import gc\n# import subprocess\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.118619Z","iopub.execute_input":"2024-06-15T15:43:26.118915Z","iopub.status.idle":"2024-06-15T15:43:26.125809Z","shell.execute_reply.started":"2024-06-15T15:43:26.118891Z","shell.execute_reply":"2024-06-15T15:43:26.124921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_x = train_x.reshape((train_x.shape[0],28,28)).to(device)\n# test_x = test_x.reshape((test_x.shape[0],28,28)).to(device)\n# train_x.shape,test_x.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.126833Z","iopub.execute_input":"2024-06-15T15:43:26.127176Z","iopub.status.idle":"2024-06-15T15:43:26.135508Z","shell.execute_reply.started":"2024-06-15T15:43:26.127125Z","shell.execute_reply":"2024-06-15T15:43:26.134679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with torch.no_grad():\n#     obj1 = train_x[10].to('cpu')\n# plt.imshow(obj1,cmap = \"gray\")\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.136555Z","iopub.execute_input":"2024-06-15T15:43:26.13687Z","iopub.status.idle":"2024-06-15T15:43:26.144396Z","shell.execute_reply.started":"2024-06-15T15:43:26.136839Z","shell.execute_reply":"2024-06-15T15:43:26.143611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_x = train_x.to(dtype = torch.float32)\n# test_x = test_x.to(dtype = torch.float32)\n# train_x = torch.unsqueeze(train_x,1)\n# test_x = torch.unsqueeze(test_x,1)","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.145393Z","iopub.execute_input":"2024-06-15T15:43:26.145671Z","iopub.status.idle":"2024-06-15T15:43:26.154922Z","shell.execute_reply.started":"2024-06-15T15:43:26.145648Z","shell.execute_reply":"2024-06-15T15:43:26.153994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_x.dtype,train_x.shape,test_x.shape","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.15605Z","iopub.execute_input":"2024-06-15T15:43:26.156341Z","iopub.status.idle":"2024-06-15T15:43:26.165378Z","shell.execute_reply.started":"2024-06-15T15:43:26.156317Z","shell.execute_reply":"2024-06-15T15:43:26.164502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalization","metadata":{}},{"cell_type":"code","source":"# train_x = torch.log(train_x+1)/math.log(256)\n# test_x = torch.log(test_x+1)/math.log(256)","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.166516Z","iopub.execute_input":"2024-06-15T15:43:26.166789Z","iopub.status.idle":"2024-06-15T15:43:26.176695Z","shell.execute_reply.started":"2024-06-15T15:43:26.166765Z","shell.execute_reply":"2024-06-15T15:43:26.175805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with torch.no_grad():\n#     obj1 = train_x[0].to('cpu')\n# plt.imshow(obj1[0],cmap = \"gray\")\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-15T15:43:26.177835Z","iopub.execute_input":"2024-06-15T15:43:26.178361Z","iopub.status.idle":"2024-06-15T15:43:26.186345Z","shell.execute_reply.started":"2024-06-15T15:43:26.178329Z","shell.execute_reply":"2024-06-15T15:43:26.185525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# making the generator","metadata":{}},{"cell_type":"code","source":"class Generator(nn.Module):\n    def __init__(self):\n        super().__init__()\n        layers = []\n        in_ch = 8\n        for i in range(3):\n            layers += [nn.ConvTranspose2d(in_ch,in_ch//2,2,2)]\n            in_ch = in_ch//2\n            layers += [nn.BatchNorm2d(in_ch)]\n            layers += [nn.LeakyReLU()]\n        self.main = nn.Sequential(*layers)\n        self.upsamp = nn.Upsample(scale_factor = 28/16 ,  mode='nearest')\n        self.activ = nn.Sigmoid()\n    def forward(self,x):\n        gen_img = self.main(x)\n        gen_img = self.upsamp(gen_img)\n        gen_img = self.activ(gen_img)\n        return gen_img","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:07.022626Z","iopub.execute_input":"2024-06-16T08:24:07.023245Z","iopub.status.idle":"2024-06-16T08:24:07.030524Z","shell.execute_reply.started":"2024-06-16T08:24:07.023215Z","shell.execute_reply":"2024-06-16T08:24:07.02956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Discriminator","metadata":{}},{"cell_type":"code","source":"class Discriminator(nn.Module):\n    def __init__(self):\n        super().__init__()\n        layers = []\n        self.localization = nn.Sequential(\n            nn.Conv2d(1, 8, kernel_size=7),\n            nn.MaxPool2d(2, stride=2),\n            nn.ReLU(True),\n            nn.Conv2d(8, 10, kernel_size=5),\n            nn.MaxPool2d(2, stride=2),\n            nn.ReLU(True)\n        )\n        # Regressor for the 3 * 2 affine matrix\n        self.fc_loc = nn.Sequential(\n            nn.Linear(10 * 3 * 3, 32),\n            nn.ReLU(True),\n            nn.Linear(32, 3 * 2)\n        )\n\n        for i in range(10):\n            layers += [nn.Conv2d(1,1,3,1)]\n            layers += [nn.BatchNorm2d(1)]\n            layers += [nn.LeakyReLU(1.5)]\n        self.main = nn.Sequential(*layers)\n        self.sig = nn.Sigmoid()\n        # Spatial transformer network forward function\n    def stn(self, x):\n        xs = self.localization(x)\n        xs = xs.view(-1, 10 * 3 * 3)\n        theta = self.fc_loc(xs)\n        theta = theta.view(-1, 2, 3)\n        grid = F.affine_grid(theta, x.size())\n        x = F.grid_sample(x, grid)\n        return x\n    \n    def forward(self,x):\n        y = self.stn(x)\n        y = self.main(x)\n        y = self.sig(y)\n#         y = y.mean(dim = (1,2,3))\n        return y\n            ","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:07.846178Z","iopub.execute_input":"2024-06-16T08:24:07.846533Z","iopub.status.idle":"2024-06-16T08:24:07.857046Z","shell.execute_reply.started":"2024-06-16T08:24:07.846505Z","shell.execute_reply":"2024-06-16T08:24:07.85622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(707)\ngen = Generator()\ntorch.manual_seed(708)\ndis = Discriminator()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:08.288806Z","iopub.execute_input":"2024-06-16T08:24:08.289629Z","iopub.status.idle":"2024-06-16T08:24:08.312131Z","shell.execute_reply.started":"2024-06-16T08:24:08.289597Z","shell.execute_reply":"2024-06-16T08:24:08.311424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen = gen.to(device)\ndis = dis.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:08.684533Z","iopub.execute_input":"2024-06-16T08:24:08.685223Z","iopub.status.idle":"2024-06-16T08:24:08.821451Z","shell.execute_reply.started":"2024-06-16T08:24:08.685191Z","shell.execute_reply":"2024-06-16T08:24:08.82047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# training loop","metadata":{}},{"cell_type":"code","source":"dis_loss = []\ngen_loss = []\ncal_error = []\ndis_test_loss = []","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:41:50.287764Z","iopub.execute_input":"2024-06-16T09:41:50.288371Z","iopub.status.idle":"2024-06-16T09:41:50.292584Z","shell.execute_reply.started":"2024-06-16T09:41:50.28834Z","shell.execute_reply":"2024-06-16T09:41:50.291693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# true_lables = torch.ones(60000,dtype = torch.float32).to(device)\n# fake_lables = torch.zeros(60000,dtype = torch.float32).to(device)\ngen_opt = torch.optim.Adam(gen.parameters(),lr = 0.001)\ndis_opt = torch.optim.Adam(dis.parameters(),lr = 0.001)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:11.563202Z","iopub.execute_input":"2024-06-16T08:24:11.563521Z","iopub.status.idle":"2024-06-16T08:24:12.747785Z","shell.execute_reply.started":"2024-06-16T08:24:11.563497Z","shell.execute_reply":"2024-06-16T08:24:12.747022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = torch.nn.BCELoss(reduction = 'mean')","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:12.749254Z","iopub.execute_input":"2024-06-16T08:24:12.749969Z","iopub.status.idle":"2024-06-16T08:24:12.754339Z","shell.execute_reply.started":"2024-06-16T08:24:12.749942Z","shell.execute_reply":"2024-06-16T08:24:12.753429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#just one random image (simillar to noise)\nprevious_stored_generated_img = torch.rand(1,1,28,28).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:13.596382Z","iopub.execute_input":"2024-06-16T08:24:13.59708Z","iopub.status.idle":"2024-06-16T08:24:13.601325Z","shell.execute_reply.started":"2024-06-16T08:24:13.597049Z","shell.execute_reply":"2024-06-16T08:24:13.600441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def train(train_step,previous_stored_generated_img):\n#     hint = 1\n#     train_gen_loss = []\n#     train_dis_loss = []\n#     test_dis_loss = []\n#     calibration_error = []\n#     alpha = 1\n#     beta = 5\n#     gamma = 10\n#     for epoch in range(train_step):\n#         gen.train()\n#         dis.train()\n#         gen_opt.zero_grad()\n        \n#         flag = False\n#         noise = torch.rand((train_x.shape[0]//2,8,2,2)).to(device)\n#         ha\n#         generated_data = gen(noise)\n        \n        \n#         previous_stored_generated_img = torch.cat((previous_stored_generated_img,generated_data),dim = 0)\n#         rand_samples_numbers = random.choices(range(previous_stored_generated_img.shape[0]) , k = train_x.shape[0]//2)\n#         rand_prev_samples = previous_stored_generated_img[rand_samples_numbers,:,:,:]\n        \n#         generator_discriminator_out = dis(generated_data)\n        \n        \n        \n#         generator_loss = loss(generator_discriminator_out,\n#                                 torch.ones_like(generator_discriminator_out))\n#         train_gen_loss.append(generator_loss)\n#         generator_loss.backward()\n#         gen_opt.step()\n        \n        \n#         dis_opt.zero_grad()\n#         true_discriminator_out = dis(train_x)\n#         true_discriminator_loss = -torch.log(true_discriminator_out).mean()\n#         generator_discriminator_out = dis(generated_data.detach())\n\n#         generator_discriminator_loss = loss(\n#         generator_discriminator_out,torch.zeros_like(generator_discriminator_out))\n\n#         prev_gen_discriminator_out = dis(rand_prev_samples.detach())\n\n#         prev_generator_discriminator_loss = loss(\n#         prev_gen_discriminator_out,torch.zeros_like(prev_gen_discriminator_out))\n\n\n\n#         discriminator_loss = (alpha*true_discriminator_loss +\n#                              beta*generator_discriminator_loss+\n#                                 gamma*prev_generator_discriminator_loss )/(alpha + beta + gamma)\n\n\n#         train_dis_loss.append(discriminator_loss)\n#         discriminator_loss.backward()\n#         dis_opt.step()\n#         torch.cuda.empty_cache()\n        \n#         with torch.inference_mode():\n#             dis.eval()\n#             gen.eval()\n            \n#             new_noise = torch.rand((test_x.shape[0],8,2,2)).to(device)\n            \n#             test_gen_imgs = gen(new_noise)\n            \n#             test_dis_out_r = dis(test_x)\n#             test_dis_loss_r = -torch.log(test_dis_out_r).mean()\n            \n#             test_dis_out_f = dis(test_gen_imgs)\n            \n#             test_dis_loss_f = loss(test_dis_out_f,torch.ones_like(test_dis_out_f))\n            \n#             dis_test_loss = (alpha*test_dis_loss_r+beta*test_dis_loss_f)/(alpha+beta)\n            \n#             test_dis_loss.append(dis_test_loss)\n            \n#             cal = (torch.abs(test_dis_out_r.mean(dim = (1,2,3)))+ torch.abs(1 - test_dis_out_f.mean(dim = (1,2,3))))/2\n#             calibration_error.append(cal.mean())\n#             torch.cuda.empty_cache()\n            \n#             if epoch%40 == 0:\n#                 print(\"epoch number: \", epoch)\n#                 print(\"generator loss: \", generator_loss)\n#                 print(\"discriminator loss: \",discriminator_loss)\n#                 print(\"dicrininator test loss: \",dis_test_loss)\n#                 print(\"calibration error: \",cal.mean())\n    \n#     return [hint,train_gen_loss,train_dis_loss,calibration_error,test_dis_loss]\n        \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:14.093065Z","iopub.execute_input":"2024-06-16T08:24:14.09366Z","iopub.status.idle":"2024-06-16T08:24:14.100689Z","shell.execute_reply.started":"2024-06-16T08:24:14.093631Z","shell.execute_reply":"2024-06-16T08:24:14.099667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport random\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\n\n# Define the dataset classes (assuming you have already defined Build_Data and Test_Data as before)\n\n# Training function\ndef train(train_step, previous_stored_generated_img, train_data_loader, test_data_loader, device):\n    hint = 1\n    train_gen_loss = []\n    train_dis_loss = []\n    test_dis_loss = []\n    calibration_error = []\n    alpha = 1\n    beta = 5\n    gamma = 10\n    start_epoch = load_checkpoint(gen, dis, gen_opt, dis_opt)\n    \n    for epoch in range(start_epoch,train_step):\n        gen.train()\n        dis.train()\n        gen_opt.zero_grad()\n        sum_gen_loss = 0\n        sum_train_dis_loss = 0\n        sum_test_dis_loss = 0\n        sum_cal_error = 0\n        # Train with training data loader\n        for train_x in train_data_loader:\n            train_x = train_x.to(device)\n            flag = False\n            noise = torch.rand((train_x.shape[0]//2, 8, 2, 2)).to(device)\n            \n            generated_data = gen(noise)\n            \n            # Limit the number of previous stored images\n            if previous_stored_generated_img.shape[0] > 20000:\n                previous_stored_generated_img = previous_stored_generated_img[-10000:]\n            \n            previous_stored_generated_img = torch.cat((previous_stored_generated_img, generated_data), dim=0)\n            rand_samples_numbers = random.choices(range(previous_stored_generated_img.shape[0]), k=train_x.shape[0]//2)\n            rand_prev_samples = previous_stored_generated_img[rand_samples_numbers, :, :, :]\n            \n            generator_discriminator_out = dis(generated_data)\n            \n            generator_loss = loss(generator_discriminator_out, torch.ones_like(generator_discriminator_out))\n            sum_gen_loss += generator_loss.item()\n            generator_loss.backward()\n            gen_opt.step()\n            \n            dis_opt.zero_grad()\n            true_discriminator_out = dis(train_x)\n            true_discriminator_loss = -torch.log(true_discriminator_out).mean()\n            generator_discriminator_out = dis(generated_data.detach())\n            \n            generator_discriminator_loss = loss(generator_discriminator_out, torch.zeros_like(generator_discriminator_out))\n            prev_gen_discriminator_out = dis(rand_prev_samples.detach())\n            prev_generator_discriminator_loss = loss(prev_gen_discriminator_out, torch.zeros_like(prev_gen_discriminator_out))\n            \n            discriminator_loss = (alpha * true_discriminator_loss +\n                                  beta * generator_discriminator_loss +\n                                  gamma * prev_generator_discriminator_loss) / (alpha + beta + gamma)\n            \n            sum_train_dis_loss += discriminator_loss.item()\n            discriminator_loss.backward()\n            dis_opt.step()\n            torch.cuda.empty_cache()\n            gc.collect()\n        with torch.inference_mode():\n            dis.eval()\n            gen.eval()\n            \n            # Evaluate with test data loader\n            for test_x in test_data_loader:\n                test_x = test_x.to(device)\n                new_noise = torch.rand((test_x.shape[0], 8, 2, 2)).to(device)\n                test_gen_imgs = gen(new_noise)\n                \n                test_dis_out_r = dis(test_x)\n                test_dis_loss_r = -torch.log(test_dis_out_r).mean()\n                \n                test_dis_out_f = dis(test_gen_imgs)\n                test_dis_loss_f = loss(test_dis_out_f, torch.ones_like(test_dis_out_f))\n                \n                dis_test_loss = (alpha * test_dis_loss_r + beta * test_dis_loss_f) / (alpha + beta)\n                \n                sum_test_dis_loss += dis_test_loss.item()\n                \n                cal = (torch.abs(test_dis_out_r.mean(dim=(1, 2, 3))) + torch.abs(1 - test_dis_out_f.mean(dim=(1, 2, 3)))) / 2\n                sum_cal_error += cal.mean().item()\n                torch.cuda.empty_cache()\n                gc.collect()\n        train_gen_loss.append(sum_gen_loss/15)\n        train_dis_loss.append(sum_train_dis_loss/15)\n        calibration_error.append(sum_cal_error/2.5)\n        test_dis_loss.append(sum_test_dis_loss/15)\n        gc.collect()       \n        if epoch % 10 == 0:\n            print(f\"epoch number: {epoch}\")\n            print(f\"generator loss: {generator_loss.item()}\")\n            print(f\"discriminator loss: {discriminator_loss.item()}\")\n            print(f\"discriminator test loss: {dis_test_loss.item()}\")\n            print(f\"calibration error: {cal.mean().item()}\")\n        if epoch% 100 == 0:\n            print(\"##################################################################################################\")\n            print(\"##################################################################################################\")\n            print(epoch,\" complected\")\n            save_checkpoint(gen,dis,gen_opt,dis_opt,epoch,\"/kaggle/working/model_checkpoints\")\n    gc.collect()\n    return [hint, train_gen_loss, train_dis_loss, calibration_error, test_dis_loss]\n\n# Assuming `gen`, `dis`, `gen_opt`, `dis_opt`, `loss`, `device`, `previous_stored_generated_img`, `train_data_loader`, and `test_data_loader` are defined elsewhere\n","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:24:16.548206Z","iopub.execute_input":"2024-06-16T08:24:16.548528Z","iopub.status.idle":"2024-06-16T08:24:16.57071Z","shell.execute_reply.started":"2024-06-16T08:24:16.548505Z","shell.execute_reply":"2024-06-16T08:24:16.569815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch = 501","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:29:43.734331Z","iopub.execute_input":"2024-06-16T08:29:43.735063Z","iopub.status.idle":"2024-06-16T08:29:43.739186Z","shell.execute_reply.started":"2024-06-16T08:29:43.73503Z","shell.execute_reply":"2024-06-16T08:29:43.738154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nparm = train(epoch,previous_stored_generated_img,train_data_loader,test_data_loader,device)\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:29:45.399831Z","iopub.execute_input":"2024-06-16T08:29:45.400486Z","iopub.status.idle":"2024-06-16T09:21:24.92074Z","shell.execute_reply.started":"2024-06-16T08:29:45.400455Z","shell.execute_reply":"2024-06-16T09:21:24.919822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:21:24.922752Z","iopub.execute_input":"2024-06-16T09:21:24.923435Z","iopub.status.idle":"2024-06-16T09:21:24.927432Z","shell.execute_reply.started":"2024-06-16T09:21:24.923392Z","shell.execute_reply":"2024-06-16T09:21:24.92656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(parm[1])","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:41:58.131819Z","iopub.execute_input":"2024-06-16T09:41:58.132148Z","iopub.status.idle":"2024-06-16T09:41:58.137808Z","shell.execute_reply.started":"2024-06-16T09:41:58.132124Z","shell.execute_reply":"2024-06-16T09:41:58.136988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(240):\n    with torch.no_grad():\n        gen_loss.append(parm[1][i])\n        dis_loss.append(parm[2][i])\n        cal_error.append(parm[3][i])\n        dis_test_loss.append(parm[4][i])","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:41:58.596464Z","iopub.execute_input":"2024-06-16T09:41:58.597235Z","iopub.status.idle":"2024-06-16T09:41:58.603729Z","shell.execute_reply.started":"2024-06-16T09:41:58.597204Z","shell.execute_reply":"2024-06-16T09:41:58.602683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.plot(gen_loss)\nplt.xlabel('epoch')\nplt.ylabel('loss')\nplt.title('generator loss')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:00.526909Z","iopub.execute_input":"2024-06-16T09:42:00.527262Z","iopub.status.idle":"2024-06-16T09:42:00.783495Z","shell.execute_reply.started":"2024-06-16T09:42:00.527235Z","shell.execute_reply":"2024-06-16T09:42:00.782585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.plot(dis_loss , label = 'train_loss' , color = 'blue')\nplt.plot(dis_test_loss , label = 'test_loss' , color = 'red')\n# Add labels and title\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Train vs Test Loss discriminator')\n\n# Add legend\nplt.legend()\n\n# Show the plot\nplt.grid(True)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:04.394333Z","iopub.execute_input":"2024-06-16T09:42:04.394962Z","iopub.status.idle":"2024-06-16T09:42:04.667669Z","shell.execute_reply.started":"2024-06-16T09:42:04.394928Z","shell.execute_reply":"2024-06-16T09:42:04.666816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.plot(cal_error)\nplt.title(\"calibration error graph\")\nplt.xlabel(\"epoch\")\nplt.ylabel(\"calibration error\")\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:10.92871Z","iopub.execute_input":"2024-06-16T09:42:10.929088Z","iopub.status.idle":"2024-06-16T09:42:11.133947Z","shell.execute_reply.started":"2024-06-16T09:42:10.929061Z","shell.execute_reply":"2024-06-16T09:42:11.133005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for test_x in test_data_loader:\n    with torch.inference_mode():\n        test_x = test_x.to(device)\n        dis_real_test = dis(test_x).mean(dim = (1,2,3))\n        dis_real_test = dis_real_test.to(\"cpu\").numpy()\n        test_noise = torch.randn((test_x.shape[0],8,2,2)).to(device)\n        dis_fake_test = dis(gen(test_noise)).mean(dim = (1,2,3))\n        dis_fake_test = dis_fake_test.to(\"cpu\").numpy()\n    break;","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:14.291974Z","iopub.execute_input":"2024-06-16T09:42:14.292309Z","iopub.status.idle":"2024-06-16T09:42:14.34899Z","shell.execute_reply.started":"2024-06-16T09:42:14.292281Z","shell.execute_reply":"2024-06-16T09:42:14.348245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting a basic histogram\nplt.hist(dis_real_test, bins=10, color='skyblue', edgecolor='black')\n \n# Adding labels and title\nplt.xlabel('Values')\nplt.ylabel('Frequency')\nplt.title('confidence value histrogram for real images')","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:15.939158Z","iopub.execute_input":"2024-06-16T09:42:15.9395Z","iopub.status.idle":"2024-06-16T09:42:16.169978Z","shell.execute_reply.started":"2024-06-16T09:42:15.939473Z","shell.execute_reply":"2024-06-16T09:42:16.16911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting a basic histogram\nplt.hist(dis_fake_test, bins=6, color='skyblue', edgecolor='black')\n \n# Adding labels and title\nplt.xlabel('Values')\nplt.ylabel('Frequency')\nplt.title('confidence value histrogram for generated images')","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:41.223378Z","iopub.execute_input":"2024-06-16T09:42:41.223707Z","iopub.status.idle":"2024-06-16T09:42:41.543401Z","shell.execute_reply.started":"2024-06-16T09:42:41.223681Z","shell.execute_reply":"2024-06-16T09:42:41.542506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.inference_mode():\n    real_prob = dis(test_x).mean(dim = (1,2,3)).to(device)\n    print(\"real probs shape\",real_prob.shape)\n    fake_prob = dis(gen(torch.rand(test_x.shape[0],8,2,2).to(device))).mean(dim = (1,2,3))\n    one_minus_real = 1-real_prob\n    one_minus_fake = 1-fake_prob\n    probs_real = torch.vstack((real_prob,one_minus_real))\n    probs_fake = torch.vstack((fake_prob,one_minus_fake))\n#     print(\"real probs dim:\", probs_real.shape)\n#     print(\"fake probs dim:\",probs_fake.shape)\n    probs_ = torch.cat((probs_real,probs_fake),dim = 1).to('cpu').numpy()\n    lables_ = torch.cat((torch.ones_like(real_prob),torch.zeros_like(fake_prob))).to(device).to('cpu').numpy()\n#     print(\"probs_ shape:\" , probs_.shape)\n#     print(\"lables shape\" , lables_.shape)\nplot_multiclass_calibration_curve(probs_.T,lables_)\nprint('end')","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:46.721697Z","iopub.execute_input":"2024-06-16T09:42:46.722039Z","iopub.status.idle":"2024-06-16T09:42:47.100923Z","shell.execute_reply.started":"2024-06-16T09:42:46.722013Z","shell.execute_reply":"2024-06-16T09:42:47.100077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# generationg fake images","metadata":{}},{"cell_type":"code","source":"# generating fake images \nnoise = torch.randn(25,8,2,2).to(device)\nwith torch.inference_mode():\n    gen_imgs = gen(noise)\ngen_imgs.shape\nplot_image_grid(gen_imgs,5,5)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:42:59.552307Z","iopub.execute_input":"2024-06-16T09:42:59.552942Z","iopub.status.idle":"2024-06-16T09:43:00.606554Z","shell.execute_reply.started":"2024-06-16T09:42:59.552908Z","shell.execute_reply":"2024-06-16T09:43:00.605711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"end\")","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:43:03.831021Z","iopub.execute_input":"2024-06-16T09:43:03.831346Z","iopub.status.idle":"2024-06-16T09:43:03.836086Z","shell.execute_reply.started":"2024-06-16T09:43:03.831322Z","shell.execute_reply":"2024-06-16T09:43:03.835183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_checkpoint(gen,dis,gen_opt,dis_opt,500,\"/kaggle/working/model_checkpoints\")","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:43:12.267298Z","iopub.execute_input":"2024-06-16T09:43:12.267634Z","iopub.status.idle":"2024-06-16T09:43:12.292149Z","shell.execute_reply.started":"2024-06-16T09:43:12.267609Z","shell.execute_reply":"2024-06-16T09:43:12.291465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"end\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"end\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}