{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Simple Dataset reading pipeline","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision, torchinfo, torchmetrics\nimport torchvision\nfrom sklearn.model_selection import train_test_split\nimport os, glob, zipfile\nfrom tqdm import tqdm\nfrom PIL import Image\n\ndef DOWNLOAD_DATASETS():\n    zip_files = ['test.zip', 'train.zip']\n\n    for zip_file in zip_files:\n        with zipfile.ZipFile(\"../input/dogs-vs-cats-redux-kernels-edition/{}\".format(zip_file),\"r\") as z:\n            z.extractall(\".\")\n            print(\"{} unzipped\".format(zip_file))\n\n    train_file_names_list = glob.glob(os.path.join(\"../working/train\",'*.jpg'))\n    test_file_names_list  = glob.glob(os.path.join(\"../working/test\", '*.jpg'))\n\n    train_list, val_list  = train_test_split(train_file_names_list, test_size=0.2)\n\n    transformation_list =  torchvision.transforms.Compose([\n            torchvision.transforms.Resize((224, 224)), # IMPORTANT SIZE OF IMAGE (224, 224) or (256,256)\n            torchvision.transforms.ToTensor(),\n        ])\n\n    class Custom_Dataset(torch.utils.data.Dataset):\n        def __init__(self,file_list,transformation_list = None):\n            self.file_list = file_list\n            self.transform = transformation_list\n\n        def __len__(self):\n            self.filelength = len(self.file_list)\n            return self.filelength\n\n        def __getitem__(self,idx):\n            img_path = self.file_list[idx]\n            img = Image.open(img_path)\n            img_transformed = self.transform(img)\n\n            label = img_path.split('/')[-1].split('.')[0]\n            if label == 'dog':\n                label=1\n            elif label == 'cat':\n                label=0\n\n            return img_transformed,label\n\n    train_dataset = Custom_Dataset(train_list, transformation_list)\n    val_dataset   = Custom_Dataset(val_list  , transformation_list)\n\n    train_loader  = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=32, shuffle=True )\n    val_loader    = torch.utils.data.DataLoader(dataset = val_dataset, batch_size=32, shuffle=True)\n    \n    return train_dataset, val_dataset, train_loader, val_loader\n\n\ntraining_dataset, validation_dataset, training_dataloader, validation_dataloader = DOWNLOAD_DATASETS();\nassert next(iter(training_dataloader)) is not None\nassert next(iter(validation_dataloader)) is not None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Simple Model Training Pipeline","metadata":{}},{"cell_type":"code","source":"lr      = 0.001 # learning_rate\nepochs  = 10 # How much to train a model\ndevice  = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef TRAIN_MODEL(model, training_dataloader, validation_dataloader):\n    \n    model.train(mode=True)\n    OPTIMIZER = torch.optim.SGD ( params= model.parameters(), lr= lr ) # Using torch.optimizer algorithm\n    metric    = torchmetrics.Accuracy(task=\"multiclass\", num_classes= 2 ).to(device)\n    \n    for epoch_no in range(epochs):\n        for batch_no, (image_tensors, labels) in enumerate(progress_bar := tqdm(training_dataloader)):\n            \n            x_actual, y_actual = image_tensors.to(device), labels.to(device)\n            \n            y_predicted_LOGITS = model.forward               (x_actual)\n            y_predicted_probs  = nn.functional.softmax       (y_predicted_LOGITS, dim= 1)\n            loss               = nn.functional.cross_entropy (y_predicted_LOGITS, y_actual.to(torch.int64))\n            \n            OPTIMIZER.zero_grad()\n            loss.backward()\n            # dError_dParameters    = torch.autograd.grad( outputs = ERROR_FUNC( y_predicted, y_actual ), inputs = model.parameters())\n            # Parameters of layer 1 are not dependent on any other parameters\n            # Parameters of layer 2 are dependent on layer 1 parameters\n            # Parameters of layer 3 are dependent on layer 2 parameters which are dependent on layer 1 parameters\n            # Finding complicated rate of change of such nested parameters is done automatically when we do loss.backward()\n            OPTIMIZER.step()\n            \"\"\"\n            for (name, weight), gradient in zip(model.named_parameters(), dError_dWeights):\n                weight = weight - gradient * LEARNING_RATE\n                print(f\"Parameters of layer: {name} have these many {torch.count_nonzero(gradient)} updates out of {torch.count(gradient)})\n            \"\"\"\n\n            loss_batch      = loss.item()\n            accuracy_batch  = metric(y_predicted_LOGITS, y_actual)\n            train_acc_epoch = metric.compute() # calculates average accuracy across epoch automatically\n\n            metrics_per_batch = {\n                \"loss_batch\": loss_batch,\n                \"accuracy_running_average\": train_acc_epoch,\n            }\n            progress_bar.set_description(f'batch_no = {batch_no},\\t loss_batch = {loss_batch:0.4f},\\t accuracy_avg = {train_acc_epoch:0.4f}')\n            \n        metric.reset()\n        loss_validation, accuracy_validation = EVALUATE_MODEL(model, validation_dataloader)\n        print(f'epoch_no = {epoch_no}, training_loss = {loss_batch:0.4f}, validation_loss = {loss_validation:0.4f},\\t training_accuracy = {accuracy_batch:0.4f}, validation_accuracy = {accuracy_validation:0.4f}')\n    \n    model.train(mode=False)\n\n\ndef EVALUATE_MODEL(model, validation_dataloader):\n    # EVALUATE MODEL AT END OF EVERY EPOCH\n    model.eval()\n    metric = torchmetrics.Accuracy(task=\"multiclass\", num_classes= 2 ).to(device)\n    with torch.no_grad():\n        for batch_no, (image_tensors, labels) in enumerate(validation_dataloader):\n            x_actual, y_actual = image_tensors.to(device), labels.to(device)\n\n            y_predicted_LOGITS = model.forward                 (x_actual)\n            loss               = nn.functional.cross_entropy   (y_predicted_LOGITS, y_actual.to(torch.int64)).item()\n            accuracy_batch     = metric                        (y_predicted_LOGITS, y_actual).item()\n\n        testing_accuracy_avg = metric.compute().item()\n    return loss, testing_accuracy_avg","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Simple Transfer Learning","metadata":{}},{"cell_type":"code","source":"import timm\n\ntest_image = torch.randn(2, 3, 224, 224)\n\npre_trained_model = timm.create_model('vgg16'                , pretrained=True, )\n\nprint(pre_trained_model)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T13:26:37.071609Z","iopub.execute_input":"2024-06-08T13:26:37.072057Z","iopub.status.idle":"2024-06-08T13:26:40.380070Z","shell.execute_reply.started":"2024-06-08T13:26:37.072023Z","shell.execute_reply":"2024-06-08T13:26:40.378854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_trained_model.head","metadata":{"execution":{"iopub.status.busy":"2024-06-08T13:26:45.071566Z","iopub.execute_input":"2024-06-08T13:26:45.071991Z","iopub.status.idle":"2024-06-08T13:26:45.079736Z","shell.execute_reply.started":"2024-06-08T13:26:45.071958Z","shell.execute_reply":"2024-06-08T13:26:45.078433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_trained_model.features","metadata":{"execution":{"iopub.status.busy":"2024-06-08T13:26:52.390973Z","iopub.execute_input":"2024-06-08T13:26:52.391353Z","iopub.status.idle":"2024-06-08T13:26:52.398917Z","shell.execute_reply.started":"2024-06-08T13:26:52.391324Z","shell.execute_reply":"2024-06-08T13:26:52.397739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_trained_model.pre_logits","metadata":{"execution":{"iopub.status.busy":"2024-06-08T13:27:03.325887Z","iopub.execute_input":"2024-06-08T13:27:03.326265Z","iopub.status.idle":"2024-06-08T13:27:03.333343Z","shell.execute_reply.started":"2024-06-08T13:27:03.326237Z","shell.execute_reply":"2024-06-08T13:27:03.332024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transfer_learning_model = nn.Sequential(\n    pre_trained_model,\n    nn.Flatten(start_dim=1),\n    nn.Dense(in_features = 512*7*7, out_features = 2),\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torchinfo.summary(transfer_learning_model, input_size= (1,3,224,224) )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Standard transfer learning models","metadata":{}},{"cell_type":"code","source":"m = timm.create_model('resnet50'             , pretrained=True)\nm = timm.create_model('densenet121'          , pretrained=True)\nm = timm.create_model('efficientnet_b0'      , pretrained=True)\nm = timm.create_model('inception_v4'         , pretrained=True)\nm = timm.create_model('mobilenetv2_100'      , pretrained=True)\nm = timm.create_model('inception_v4'         , pretrained=True)\nm = timm.create_model('inception_v4'         , pretrained=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = torch.hub.load('pytorch/vision:v0.10.0', 'alexnet',       pretrained=True)\nmodel = torch.hub.load('pytorch/vision:v0.10.0', 'squeezenet1_0', pretrained=True) # Alexnet level accuracy with 50x fewer parameters\nmodel = torch.hub.load(\"coderx7/simplenet_pytorch:v1.0.0\", \"simplenetv1_5m_m1\", pretrained=True)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T13:10:53.381470Z","iopub.execute_input":"2024-06-08T13:10:53.381856Z","iopub.status.idle":"2024-06-08T13:10:57.606880Z","shell.execute_reply.started":"2024-06-08T13:10:53.381824Z","shell.execute_reply":"2024-06-08T13:10:57.605691Z"},"trusted":true},"execution_count":null,"outputs":[]}]}