{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle Link: https://www.kaggle.com/code/skrymr/mldl-cifar10\n\nWandb Link: https://wandb.ai/skrymr/MLDL_Cifar","metadata":{}},{"cell_type":"markdown","source":"# try:\n    import py7zr\nexcept:\n    !pip install py7zr\n    import py7zr\n#!python -m py7zr x /kaggle/input/cifar-10/train.7z\n#!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:05:23.481817Z","iopub.execute_input":"2024-01-22T17:05:23.482104Z","iopub.status.idle":"2024-01-22T17:05:39.653412Z","shell.execute_reply.started":"2024-01-22T17:05:23.482077Z","shell.execute_reply":"2024-01-22T17:05:39.652376Z"}}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk(\"/kaggle/input\"):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\ndata_dir = \"/kaggle/working/\"\nif not (os.path.exists(\"/kaggle/working/train\") and os.path.isdir(\"/kaggle/working/train\")):\n    with py7zr.SevenZipFile(\"/kaggle/input/cifar-10/train.7z\", mode=\"r\") as z:\n        z.extractall()\n\nif not (os.path.exists(\"/kaggle/working/test\") and os.path.isdir(\"/kaggle/working/test\")):\n    with py7zr.SevenZipFile(\"/kaggle/input/cifar-10/test.7z\", mode=\"r\") as z:\n        z.extractall()\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-22T17:05:39.655504Z","iopub.execute_input":"2024-01-22T17:05:39.655839Z","iopub.status.idle":"2024-01-22T17:25:24.574394Z","shell.execute_reply.started":"2024-01-22T17:05:39.655810Z","shell.execute_reply":"2024-01-22T17:25:24.573343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nimport time\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms.v2 as transforms\nfrom torchvision import datasets\nfrom torchinfo import summary\n\nimport wandb\n\nconfig = {\"img_channels\": 3,\n          \"width\": 224, # Chosen by the authors of the ResNet18 PyTorch model\n          \"height\": 224,# Chosen by the authors of the ResNet18 PyTorch model\n          \"batch_size\": 64,\n          \"epochs\": 6,\n          \"optimizer\": \"SGD\",\n          \"learning_rate\": 0.01,\n          \"decay\": 0.1,\n          \"momentum\": 0.9,\n          \"freeze\": False, # Freeze the original model's weights and only update the fully connected layer\n          \"net\": {\n              \"linear_layers\": [128, 10],\n              \"activation\": \"relu\",\n              \"dropout\": 0.5\n          },\n          \"preprocess\": True,\n          \"logging\": \"standard\",\n          \"test\": False,\n          \"submission\": False}\n\nrun = wandb.init(project=\"MLDL_Cifar\",\n                 name=\"Removed freezing\",\n                 notes=\"Freezing the model reduces performance drastically.\",\n                 mode=\"online\" if not (config[\"submission\"] or config[\"test\"]) else \"disabled\",\n                 config=config,\n                 save_code=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:25:24.575653Z","iopub.execute_input":"2024-01-22T17:25:24.576075Z","iopub.status.idle":"2024-01-22T17:26:27.685164Z","shell.execute_reply.started":"2024-01-22T17:25:24.576049Z","shell.execute_reply":"2024-01-22T17:26:27.684233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, \"r\") as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(\",\") for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, \"/kaggle/input/cifar-10/trainLabels.csv\"))\nprint(f\"Number training examples: {len(labels)}\")\nprint(f\"Number classes: {len(set(labels.values()))}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:26:27.686321Z","iopub.execute_input":"2024-01-22T17:26:27.686631Z","iopub.status.idle":"2024-01-22T17:26:28.285487Z","shell.execute_reply.started":"2024-01-22T17:26:27.686579Z","shell.execute_reply":"2024-01-22T17:26:28.284578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(labels.values());","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:26:28.288111Z","iopub.execute_input":"2024-01-22T17:26:28.288403Z","iopub.status.idle":"2024-01-22T17:26:29.277466Z","shell.execute_reply.started":"2024-01-22T17:26:28.288378Z","shell.execute_reply":"2024-01-22T17:26:29.276612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio = 0.1):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, \"train\")):\n        label = labels[train_file.split(\".\")[0]]\n        fname = os.path.join(data_dir, \"train\", train_file)\n        copyfile(fname, os.path.join(data_dir, \"train_valid_test\",\n                                     \"train_valid\", label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, \"train_valid_test\",\n                                         \"valid\", label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, \"train_valid_test\",\n                                         \"train\", label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, \"test\")):\n        copyfile(os.path.join(data_dir, \"test\", test_file),\n                 os.path.join(data_dir, \"train_valid_test\", \"test\",\n                              \"unknown\"))\n        \ndef reorg_cifar10_data(data_dir, labels, valid_ratio = 0.1):\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:26:29.279005Z","iopub.execute_input":"2024-01-22T17:26:29.279354Z","iopub.status.idle":"2024-01-22T17:26:29.761990Z","shell.execute_reply.started":"2024-01-22T17:26:29.279322Z","shell.execute_reply":"2024-01-22T17:26:29.761058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"preprocess\"]:\n    reorg_cifar10_data(data_dir, labels)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:26:29.763135Z","iopub.execute_input":"2024-01-22T17:26:29.763417Z","iopub.status.idle":"2024-01-22T17:27:30.878377Z","shell.execute_reply.started":"2024-01-22T17:26:29.763393Z","shell.execute_reply":"2024-01-22T17:27:30.877512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours! \n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{}},{"cell_type":"code","source":"# Appropriate transforms for the model are already defined in its weights property\ntransform_train = transforms.Compose([\n    transforms.ToImageTensor(),\n    transforms.Normalize([0.4914, 0.4822, 0.4465],\n                         [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = transforms.Compose([\n    transforms.ToImageTensor(),\n    transforms.Normalize([0.4914, 0.4822, 0.4465],\n                         [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:27:30.879619Z","iopub.execute_input":"2024-01-22T17:27:30.879988Z","iopub.status.idle":"2024-01-22T17:27:31.456047Z","shell.execute_reply.started":"2024-01-22T17:27:30.879956Z","shell.execute_reply":"2024-01-22T17:27:31.455186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def AddLayers(model, out_features_list, activation = \"relu\", dropout = 0, freeze = False):\n    if freeze:\n        for param in model.parameters():\n            param.requires_grad = False\n    fc = model.fc\n    layers = []\n    in_features = fc.in_features # in_features = 512\n    for out_features in out_features_list[:-1]:\n        layers.append(nn.Linear(in_features, out_features))\n        if activation == \"leaky_relu\":\n            layers.append(nn.LeakyReLU())\n        else:\n            layers.append(nn.ReLU())\n        if dropout > 0:\n            layers.append(nn.Dropout(dropout))\n        in_features = out_features\n    layers.append(nn.Linear(in_features, out_features_list[-1]))\n    model.fc = nn.Sequential(*layers)\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:27:31.457676Z","iopub.execute_input":"2024-01-22T17:27:31.458244Z","iopub.status.idle":"2024-01-22T17:27:32.072788Z","shell.execute_reply.started":"2024-01-22T17:27:31.458210Z","shell.execute_reply":"2024-01-22T17:27:32.071548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.models import resnet18, ResNet18_Weights\n\nweights = ResNet18_Weights.DEFAULT\nmodel = AddLayers(resnet18(weights), config[\"net\"][\"linear_layers\"], config[\"net\"][\"activation\"], config[\"net\"][\"dropout\"], config[\"freeze\"])\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\ntrain_ds, train_valid_ds = [datasets.ImageFolder(\n    os.path.join(data_dir, \"train_valid_test\", folder),\n    transform=weights.transforms()) for folder in [\"train\", \"train_valid\"]]\n\nvalid_ds, test_ds = [datasets.ImageFolder(\n    os.path.join(data_dir, \"train_valid_test\", folder),\n    transform=weights.transforms()) for folder in [\"valid\", \"test\"]]\n\nassert train_ds.class_to_idx == train_valid_ds.class_to_idx, \"Class indices do not match!\"\nassert train_ds.class_to_idx == valid_ds.class_to_idx, \"Class indices do not match!\"\n\nclasses = {val: key for key, val in train_ds.class_to_idx.items()}\n\nworkers = os.cpu_count()\n\ntrain_loader, train_val_loader = [torch.utils.data.DataLoader(\n    dataset, config[\"batch_size\"], shuffle=True, drop_last=True, num_workers=workers)\n    for dataset in (train_ds, train_valid_ds)]\n\nval_loader = torch.utils.data.DataLoader(valid_ds, config[\"batch_size\"], shuffle=False,\n                                         drop_last=True, num_workers=workers)\n\ntest_loader = torch.utils.data.DataLoader(test_ds, config[\"batch_size\"], shuffle=False,\n                                          drop_last=False, num_workers=workers)\n\nsummary(model, input_size=(config[\"batch_size\"], config[\"img_channels\"], config[\"width\"], config[\"height\"]))","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:27:32.074395Z","iopub.execute_input":"2024-01-22T17:27:32.074779Z","iopub.status.idle":"2024-01-22T17:27:36.085044Z","shell.execute_reply.started":"2024-01-22T17:27:32.074734Z","shell.execute_reply":"2024-01-22T17:27:36.083812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{}},{"cell_type":"code","source":"def inspect_grad(model, output=False, prnt=False):\n    layers = []\n    names = []\n    for name, layer in model.named_parameters():\n        if name.endswith(\"weight\"):\n            names.append(name.strip(\".weight\"))\n            layers.append(layer)\n\n    means = []\n    maxes = []\n    gradients = {}\n    for i, (name, layer) in enumerate(zip(names, layers)):\n        if layer.grad == None:\n            continue\n        mean = layer.grad.mean().item()\n        mx = layer.grad.max().item()\n        means.append(mean)\n        maxes.append(mx)\n        gradients[name] = {\"mean\": mean, \"max\": mx}\n        if prnt:\n            print(f\"{name}:\")\n            print(f\"- mean: {mean}\")\n            print(f\"- max: {mx}\")\n            \n    if output:\n        wandb.log({\"gradients\": gradients}, commit=False)\n        \n    return max(means), sum(maxes) / len(maxes)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:27:36.088199Z","iopub.execute_input":"2024-01-22T17:27:36.088609Z","iopub.status.idle":"2024-01-22T17:27:36.680450Z","shell.execute_reply.started":"2024-01-22T17:27:36.088554Z","shell.execute_reply":"2024-01-22T17:27:36.679406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"optimizer\"] == \"Adam\":\n    optim = torch.optim.Adam(model.parameters(), lr=config[\"learning_rate\"], weight_decay=config[\"decay\"])\nelif config[\"optimizer\"] == \"SGDm\":\n    optim = torch.optim.SGD(model.parameters(), lr=config[\"learning_rate\"], momentum=config[\"momentum\"])\nelse:\n    optim = torch.optim.SGD(model.parameters(), lr=config[\"learning_rate\"])\n\ncriterion = nn.CrossEntropyLoss()\nstart_time = time.time()\nwith tqdm(total=config[\"epochs\"], position=1, leave=True) as pbar:\n    for epoch in range(config[\"epochs\"]):\n        running_loss = 0.0\n        running_acc = 0.0\n        model.train()\n        for i, (imgs, labels) in enumerate(tqdm(train_loader, position=0, leave=True)):\n            optim.zero_grad()\n            imgs, labels = imgs.to(device), labels.to(device)\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optim.step()\n            with torch.no_grad():\n                current_loss = loss.cpu().item()\n                running_loss += current_loss\n                predictions = torch.argmax(outputs, dim=1)\n                current_acc = (predictions == labels).cpu().float().mean().item()\n                running_acc += current_acc\n                if \"verbose\" in config[\"logging\"].split(\",\"):\n                    wandb.log({\"training\": {\"pointwise_loss\": current_loss,\n                                            \"pointwise_acc\": current_acc,\n                                            \"point\": i}}, commit=False)\n                    \n        wandb.log({\"training\": {\"loss\": running_loss / (i+1),\n                                \"accuracy\": running_acc / (i+1)}}, commit=False)\n        max_of_means, mean_of_maxes = inspect_grad(model,\n                                                   output=False if \"verbose\" not in config[\"logging\"].split(\",\") else True,\n                                                   prnt=False if \"debug\" not in config[\"logging\"].split(\",\") else True)\n        wandb.log({\"gradient\": {\"max_mean\": max_of_means,\n                                \"average_max\": mean_of_maxes}}, commit=False)\n            \n        running_loss = 0.0\n        running_acc = 0.0\n        model.eval()\n        with torch.no_grad():\n            for i, (imgs, labels) in enumerate(tqdm(train_val_loader, position=0, leave=True)):\n                imgs, labels = imgs.to(device), labels.to(device)\n                outputs = model(imgs)\n                loss = criterion(outputs, labels)\n                current_loss = loss.cpu().item()\n                running_loss += current_loss\n                predictions = torch.argmax(outputs, dim=1)\n                current_acc = (predictions == labels).cpu().float().mean().item()\n                running_acc += current_acc\n                if \"verbose\" in config[\"logging\"].split(\",\"):\n                    wandb.log({\"validation\": {\"pointwise_loss\": current_loss,\n                                              \"pointwise_acc\": current_acc,\n                                              \"point\": i}}, commit=False)\n        wandb.log({\"validation\": {\"loss\": running_loss / (i+1),\n                                  \"accuracy\": running_acc / (i+1)}}, commit=True)\n        torch.save(model.state_dict(), f\"/kaggle/working/model.pt\")\n        pbar.update()\n        \nwandb.log({\"total_training_time\": time.time() - start_time})","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:27:36.685547Z","iopub.execute_input":"2024-01-22T17:27:36.688079Z","iopub.status.idle":"2024-01-22T17:43:15.794870Z","shell.execute_reply.started":"2024-01-22T17:27:36.688040Z","shell.execute_reply":"2024-01-22T17:43:15.793866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    model.eval()\n    with torch.no_grad():\n        preds = []\n        for imgs, labels in tqdm(test_loader):\n            output = torch.argmax(model(imgs.to(device)), dim=1)\n            preds.extend([classes[label.item()] for label in torch.argmax(model(imgs.to(device)), dim=1).to(torch.int).cpu()])\n    out_df = pd.DataFrame(np.c_[[int(os.path.basename(name[0]).rstrip(\".png\")) for name in test_ds.samples], preds], columns=[\"id\", \"label\"])\n    out_df.sort_values(\"id\")\n    out_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T18:31:23.764301Z","iopub.execute_input":"2024-01-22T18:31:23.764696Z","iopub.status.idle":"2024-01-22T18:38:31.656426Z","shell.execute_reply.started":"2024-01-22T18:31:23.764660Z","shell.execute_reply":"2024-01-22T18:38:31.655272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T17:50:53.387011Z","iopub.execute_input":"2024-01-22T17:50:53.387374Z","iopub.status.idle":"2024-01-22T17:50:58.490043Z","shell.execute_reply.started":"2024-01-22T17:50:53.387331Z","shell.execute_reply":"2024-01-22T17:50:58.489302Z"},"trusted":true},"execution_count":null,"outputs":[]}]}