{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Simple Dataset Download","metadata":{}},{"cell_type":"code","source":"import torch, torch.nn as nn\nimport torchvision, torchinfo, torchmetrics\nimport datasets as huggingface_datasets\nfrom tqdm import tqdm\n\ndevice        = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nBATCH_SIZE    = 5\n# IMAGE_RESIZE  = 28,28\n\ndef DOWNLOAD_DATASETS():\n    # Download\n    dataset            = huggingface_datasets.load_dataset(\"cifar10\", ) # streaming = True)\n    training_dataset   = dataset['train']\n    validation_dataset = dataset['test']\n\n    # Transform\n    transformations_group = torchvision.transforms.Compose([\n        torchvision.transforms.ToTensor(), # Converts every pixel into value between 0 & 1. \n        # torchvision.transforms.Resize(size=config.IMAGE_RESIZE)\n    ])\n\n    def transform_datasets(examples):\n        examples[\"image_tensors\"] = []\n\n        for image in examples['img']:\n            transformed_image = transformations_group(image)\n            examples['image_tensors'].append(transformed_image)\n\n        return examples\n\n    training_dataset       = training_dataset   .map(transform_datasets  , batched= True)\n    validation_dataset     = validation_dataset .map(transform_datasets  , batched= True)\n\n    # Convert\n    new_training_dataset   = training_dataset   .with_format(\"torch\", columns=['label', 'image_tensors'], dtype = torch.float32)\n    new_validation_dataset = validation_dataset .with_format(\"torch\", columns=['label', 'image_tensors'], dtype = torch.float32)\n\n    TOTAL_BATCHES = len(training_dataset) / BATCH_SIZE\n    \n    training_dataloader   = torch.utils.data.DataLoader( dataset= new_training_dataset   , batch_size= BATCH_SIZE, shuffle= True )\n    validation_dataloader = torch.utils.data.DataLoader( dataset= new_validation_dataset , batch_size= BATCH_SIZE, shuffle= True )\n    \n    return training_dataset, validation_dataset, training_dataloader, validation_dataloader\n\ntraining_dataset, validation_dataset, training_dataloader, validation_dataloader = DOWNLOAD_DATASETS();\nassert next(iter(training_dataloader)) is not None\nassert next(iter(validation_dataloader)) is not None","metadata":{"execution":{"iopub.status.busy":"2024-06-07T13:39:37.524245Z","iopub.execute_input":"2024-06-07T13:39:37.524691Z","iopub.status.idle":"2024-06-07T13:41:46.529457Z","shell.execute_reply.started":"2024-06-07T13:39:37.524649Z","shell.execute_reply":"2024-06-07T13:41:46.528313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Simple Model Training Pipeline","metadata":{}},{"cell_type":"code","source":"lr      = 0.001 # learning_rate\nepochs  = 10 # How much to train a model\ndevice  = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef TRAIN_MODEL(model, training_dataloader, validation_dataloader):\n\n    model.train(mode=True)\n    OPTIMIZER = torch.optim.SGD ( params= model.parameters(), lr= lr ) # Using torch.optimizer algorithm\n    metric    = torchmetrics.Accuracy(task=\"multiclass\", num_classes= 10 ).to(device)\n    \n    for epoch_no in range(epochs):        \n        for batch_no, batch_dictionary in enumerate(progress_bar := tqdm(training_dataloader)):\n            x_actual = batch_dictionary['image_tensors'].to(device)\n            y_actual = batch_dictionary['label'].to(device)\n\n            y_predicted_LOGITS = model.forward               (x_actual)\n            y_predicted_probs  = nn.functional.softmax       (y_predicted_LOGITS, dim= 1)\n            loss               = nn.functional.cross_entropy (y_predicted_LOGITS, y_actual.to(torch.int64))\n            \n            OPTIMIZER.zero_grad()\n            loss.backward()\n            # dError_dParameters    = torch.autograd.grad( outputs = ERROR_FUNC( y_predicted, y_actual ), inputs = model.parameters())\n            # Parameters of layer 1 are not dependent on any other parameters\n            # Parameters of layer 2 are dependent on layer 1 parameters\n            # Parameters of layer 3 are dependent on layer 2 parameters which are dependent on layer 1 parameters\n            # Finding complicated rate of change of such nested parameters is done automatically when we do loss.backward()\n            OPTIMIZER.step()\n            \"\"\"\n            for (name, weight), gradient in zip(model.named_parameters(), dError_dWeights):\n                weight = weight - gradient * LEARNING_RATE\n                print(f\"Parameters of layer: {name} have these many {torch.count_nonzero(gradient)} updates out of {torch.count(gradient)})\n            \"\"\"\n\n            loss_batch      = loss.item()\n            accuracy_batch  = metric(y_predicted_LOGITS, y_actual)\n            training_accuracy_avg_epoch = metric.compute() # calculates average accuracy across epoch automatically\n\n            metrics_per_batch = {\n                \"loss_batch\": loss_batch,\n                \"accuracy_running_average\": training_accuracy_avg_epoch,\n            }\n            progress_bar.set_description(f'batch_no = {batch_no},\\t loss_batch = {loss_batch:0.4f},\\t accuracy_avg = {training_accuracy_avg_epoch:0.4f}')\n\n        metric.reset()\n        \n        loss_validation, accuracy_validation = EVALUATE_MODEL(model, validation_dataloader)\n        print(f'epoch_no = {epoch_no}, training_loss = {loss_batch:0.4f}, validation_loss = {loss_validation:0.4f},\\t training_accuracy = {accuracy_batch:0.4f}, validation_accuracy = {accuracy_validation:0.4f}')\n        model.train(mode=False)\n\ndef EVALUATE_MODEL(model, validation_dataloader):\n    model.eval()\n    metric = torchmetrics.Accuracy(task=\"multiclass\", num_classes=10).to(device)\n    with torch.no_grad():\n        for batch_no, batch_dictionary in enumerate(validation_dataloader):\n            x_actual = batch_dictionary['image_tensors'].to(device)\n            y_actual = batch_dictionary['label'].to(device)\n\n            y_predicted_LOGITS = model.forward                 (x_actual)\n            loss               = nn.functional.cross_entropy   (y_predicted_LOGITS, y_actual.to(torch.int64)).item()\n            accuracy_batch     = metric                        (y_predicted_LOGITS, y_actual).item()\n\n        testing_accuracy_avg = metric.compute().item()\n        return loss, testing_accuracy_avg","metadata":{"execution":{"iopub.status.busy":"2024-06-07T13:46:20.495450Z","iopub.execute_input":"2024-06-07T13:46:20.496431Z","iopub.status.idle":"2024-06-07T13:46:20.514419Z","shell.execute_reply.started":"2024-06-07T13:46:20.496390Z","shell.execute_reply":"2024-06-07T13:46:20.513028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Simple Model Architecture","metadata":{}},{"cell_type":"markdown","source":"## Experiment 1","metadata":{}},{"cell_type":"code","source":"# FORMATS 2d: Batch, Channels, H, W\n# FORMATS 1d: Batch, Dim\n\nmodel_random_parameters = torch.nn.Sequential(\n    \n    torch.nn.Flatten(start_dim=1),         # Dim:BCHW -> (0:B , 1:C, 2:H, 3:W)\n\n    torch.nn.Linear(in_features = 32*32*3  , out_features   = 40   ), torch.nn.ReLU(),                 # LAYER 1: 1st Hidden Layer\n    torch.nn.Linear(in_features = 40       , out_features = 30   ), torch.nn.ReLU(),                 # LAYER 2: 2nd Hidden Layer\n\n    torch.nn.Linear(in_features = 30       , out_features = 10   ),                                  # OUTPUT LAYER\n)\n\nmodel = model_random_parameters\nmodel = model.to(device)                    # Model Size / Number of Parameters are important\n\ntorchinfo.summary(model, input_size= (1,3*32*32), verbose=2);","metadata":{"execution":{"iopub.status.busy":"2024-06-07T13:42:17.696645Z","iopub.execute_input":"2024-06-07T13:42:17.697059Z","iopub.status.idle":"2024-06-07T13:42:17.733139Z","shell.execute_reply.started":"2024-06-07T13:42:17.697024Z","shell.execute_reply":"2024-06-07T13:42:17.731552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_MODEL (model, training_dataloader, validation_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-06-07T13:46:26.105336Z","iopub.execute_input":"2024-06-07T13:46:26.105776Z","iopub.status.idle":"2024-06-07T13:52:04.166601Z","shell.execute_reply.started":"2024-06-07T13:46:26.105741Z","shell.execute_reply":"2024-06-07T13:52:04.164051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Experiment 2: Convolution Neural Network","metadata":{}},{"cell_type":"code","source":"# CONVOLUTION DATA FORMAT: Batch Size,Channels ,Height , Width\nmodel = nn.Sequential(\n    \n    nn.Conv2d(in_channels = 3,  out_channels  = 40,  kernel_size = (16,16) ), nn.ReLU(),\n    nn.Conv2d(in_channels = 40,  out_channels = 40,  kernel_size = (16,16) ), nn.ReLU(),\n    \n    nn.Flatten(start_dim = 1),\n    \n    torch.nn.Linear(in_features = 40*2*2       , out_features = 10   ),\n    \n)\nmodel = model.to(device)  # Model Size / Number of Parameters are important\ntorchinfo.summary(model, input_size=(1,3,32,32), verbose=2 , col_names=[\"input_size\", \"output_size\", \"kernel_size\", \"num_params\", \"params_percent\"] );","metadata":{"execution":{"iopub.status.busy":"2024-06-05T13:14:11.761357Z","iopub.execute_input":"2024-06-05T13:14:11.762310Z","iopub.status.idle":"2024-06-05T13:14:11.780556Z","shell.execute_reply.started":"2024-06-05T13:14:11.762268Z","shell.execute_reply":"2024-06-05T13:14:11.779126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_MODEL (model, training_dataloader, validation_dataloader)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Experiment 3: Better Convolution Neural Network","metadata":{}},{"cell_type":"code","source":"feature_extractor = nn.Sequential(\n    nn.Conv2d    ( in_channels = 3, out_channels = 50,   kernel_size = (3,3) , padding=\"same\"), # input = 3, 32, 32, output: (50, 32, 32)\n    nn.ReLU(),\n    nn.MaxPool2d ( kernel_size=(2,2), stride = 2),                                              # input: (50, 32, 32) -> output: (50, 16, 16),\n    \n    nn.Conv2d    ( in_channels = 50, out_channels = 100, kernel_size = (3,3), padding=\"same\"),  # input: (50, 16, 16) -> output: (100, 16, 16),\n    nn.ReLU(),\n    nn.MaxPool2d ( kernel_size=(2,2), stride = 2),                                              # input: (100, 16, 16) -> output: (100, 8, 8),\n    \n)\n# feature_map = 100, 8, 8\n\ndecision_maker = nn.Sequential(\n    nn.Flatten(start_dim=1),\n    nn.Linear(in_features = 100*8*8    , out_features = 50), nn.ReLU(),\n    nn.Linear(in_features = 50     , out_features = 10)\n)\n\nmodel = nn.Sequential(\n  feature_extractor,\n  decision_maker\n)\n\ntest_output_logits = feature_extractor(torch.randn((1,3,32,32)))\nprint(test_output_logits.shape)\ntorchinfo.summary(model, input_size=(1,3,32,32), verbose=2 , col_names=[\"input_size\", \"output_size\", \"kernel_size\", \"num_params\", \"params_percent\"] );","metadata":{"execution":{"iopub.status.busy":"2024-06-05T13:15:41.919064Z","iopub.execute_input":"2024-06-05T13:15:41.919488Z","iopub.status.idle":"2024-06-05T13:15:41.950272Z","shell.execute_reply.started":"2024-06-05T13:15:41.919455Z","shell.execute_reply":"2024-06-05T13:15:41.949113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_MODEL (model, training_dataloader, validation_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T05:25:25.910565Z","iopub.execute_input":"2024-05-26T05:25:25.911380Z","iopub.status.idle":"2024-05-26T05:25:25.917156Z","shell.execute_reply.started":"2024-05-26T05:25:25.911348Z","shell.execute_reply":"2024-05-26T05:25:25.916230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Colored Images t0 Real Life Data - Imagenet & Model\n","metadata":{}},{"cell_type":"code","source":"import timm\n# timm.list_models(\"*vgg*\")\nmodel = timm.create_model(\"vgg11\", pretrained = True)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T06:21:21.916586Z","iopub.execute_input":"2024-05-26T06:21:21.917464Z","iopub.status.idle":"2024-05-26T06:21:54.248432Z","shell.execute_reply.started":"2024-05-26T06:21:21.917430Z","shell.execute_reply":"2024-05-26T06:21:54.247586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2024-05-26T06:23:31.986511Z","iopub.execute_input":"2024-05-26T06:23:31.987306Z","iopub.status.idle":"2024-05-26T06:23:31.993722Z","shell.execute_reply.started":"2024-05-26T06:23:31.987276Z","shell.execute_reply":"2024-05-26T06:23:31.992765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.features(torch.randn((1,3,224,224))).shape","metadata":{"execution":{"iopub.status.busy":"2024-05-26T06:25:16.019994Z","iopub.execute_input":"2024-05-26T06:25:16.020370Z","iopub.status.idle":"2024-05-26T06:25:16.138830Z","shell.execute_reply.started":"2024-05-26T06:25:16.020340Z","shell.execute_reply":"2024-05-26T06:25:16.137865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Work in Progress\n\n# !pip install py7zr\n# import py7zr\n\n# with py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z' , mode='r') as z:\n#     z.extractall(\"/kaggle/working/\")\n    \n# from torch.utils.tensorboard import SummaryWriter\n# writer = SummaryWriter()\n# writer.add_scalar('Loss/train', np.random.random(), n_iter)\n# writer.add_scalar('Loss/test', np.random.random(), n_iter)\n# writer.add_scalar('Accuracy/train', np.random.random(), n_iter)\n# writer.add_scalar('Accuracy/test', np.random.random(), n_iter)\n\n# import torch, torchvision\n# import datasets\n\n# datasets.utils.logging.set_verbosity(datasets.logging.DEBUG)\n# cifar10 = datasets.load_dataset(\"imagefolder\", data_dir=\"/kaggle/working/test\", drop_labels=True)\n# dataset = torchvision.datasets.ImageFolder(\"/kaggle/working/test\", )\n\n\n# import os\n# from PIL import Image\n\n# transformation_list = torchvision.transforms.Compose([\n# torchvision.transforms.ToTensor(),\n# torchvision.transforms.Resize(size=(224,224))\n# ])\n\n# submission_file = open(\"submission.csv\", \"w\")\n# submission_file.write(\"id,label\")\n\n# in_memory_data = torch.randn(size=(10000,3,32,32))\n# index = 0\n# for name in os.listdir(\"/kaggle/working/test\"):\n#     image_with_path = \"/kaggle/working/test/\" + name\n#     image = Image.open(image_with_path)    \n#     transformed_image = transformation_list(image)\n    \n#     in_memory_data[index] = transformed_image\n#     index = index + 1\n\n","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}