{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-02T07:29:38.855339Z","iopub.execute_input":"2024-07-02T07:29:38.855747Z","iopub.status.idle":"2024-07-02T07:29:39.304983Z","shell.execute_reply.started":"2024-07-02T07:29:38.855715Z","shell.execute_reply":"2024-07-02T07:29:39.304001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:29:40.310364Z","iopub.execute_input":"2024-07-02T07:29:40.311359Z","iopub.status.idle":"2024-07-02T07:29:59.741667Z","shell.execute_reply.started":"2024-07-02T07:29:40.311310Z","shell.execute_reply":"2024-07-02T07:29:59.740329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\nfrom datasets import Array3D, ClassLabel, Features, Image, load_dataset\nfrom matplotlib import pyplot\nfrom numpy import inf\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom torchinfo import summary\nfrom tqdm import tqdm\nfrom transformers import AdamW, ViTFeatureExtractor, ViTModel","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:17.833158Z","iopub.execute_input":"2024-07-02T07:30:17.833963Z","iopub.status.idle":"2024-07-02T07:30:36.271287Z","shell.execute_reply.started":"2024-07-02T07:30:17.833930Z","shell.execute_reply":"2024-07-02T07:30:36.270163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:36.273322Z","iopub.execute_input":"2024-07-02T07:30:36.274028Z","iopub.status.idle":"2024-07-02T07:30:36.279026Z","shell.execute_reply.started":"2024-07-02T07:30:36.273994Z","shell.execute_reply":"2024-07-02T07:30:36.277900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, test_ds = load_dataset('cifar10', split=['train[:10000]', 'test[:4000]'])\n# split up training into training + validation\nsplits = train_ds.train_test_split(test_size=0.3)\ntrain_ds = splits['train']\nval_ds = splits['test']","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:36.280537Z","iopub.execute_input":"2024-07-02T07:30:36.280969Z","iopub.status.idle":"2024-07-02T07:30:40.877275Z","shell.execute_reply.started":"2024-07-02T07:30:36.280932Z","shell.execute_reply":"2024-07-02T07:30:40.876360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, test_sub = load_dataset('cifar10', split=['train[:10000]', 'test[:10000]'])","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:40.879885Z","iopub.execute_input":"2024-07-02T07:30:40.880238Z","iopub.status.idle":"2024-07-02T07:30:42.430196Z","shell.execute_reply.started":"2024-07-02T07:30:40.880209Z","shell.execute_reply":"2024-07-02T07:30:42.429090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_extractor = ViTFeatureExtractor.from_pretrained('google/vit-base-patch16-224-in21k')","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:42.431408Z","iopub.execute_input":"2024-07-02T07:30:42.431737Z","iopub.status.idle":"2024-07-02T07:30:42.556642Z","shell.execute_reply.started":"2024-07-02T07:30:42.431708Z","shell.execute_reply":"2024-07-02T07:30:42.555484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chk = train_ds[0]\na = np.array(chk['img'])\nprint(a.shape)\ncats = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n\n\nprint(cats[chk['label']])\npyplot.imshow(a, cmap=pyplot.get_cmap('gray'))","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:42.558229Z","iopub.execute_input":"2024-07-02T07:30:42.558601Z","iopub.status.idle":"2024-07-02T07:30:42.898687Z","shell.execute_reply.started":"2024-07-02T07:30:42.558570Z","shell.execute_reply":"2024-07-02T07:30:42.897605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_images(examples):\n    images = examples['img']\n    images = [np.array(image, dtype=np.uint8) for image in images]\n    images = [np.moveaxis(image, source=-1, destination=0) for image in images]\n    inputs = feature_extractor(images=images)\n    examples['pixel_values'] = inputs['pixel_values']\n    return examples","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:42.900097Z","iopub.execute_input":"2024-07-02T07:30:42.900414Z","iopub.status.idle":"2024-07-02T07:30:42.907206Z","shell.execute_reply.started":"2024-07-02T07:30:42.900386Z","shell.execute_reply":"2024-07-02T07:30:42.906002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = Features({\n    'label': ClassLabel(names=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']),\n    'img': Image(decode=True, id=None),\n    'pixel_values': Array3D(dtype=\"float32\", shape=(3, 224, 224)), \n})\n\npreprocessed_train_ds = train_ds.map(preprocess_images, batched=True, features=features)\npreprocessed_val_ds = val_ds.map(preprocess_images, batched=True, features=features)\npreprocessed_test_ds = test_ds.map(preprocess_images, batched=True, features=features)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:30:42.908491Z","iopub.execute_input":"2024-07-02T07:30:42.908895Z","iopub.status.idle":"2024-07-02T07:32:14.947787Z","shell.execute_reply.started":"2024-07-02T07:30:42.908856Z","shell.execute_reply":"2024-07-02T07:32:14.946647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_test_sub = test_sub.map(preprocess_images, batched=True, features=features)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:32:14.949146Z","iopub.execute_input":"2024-07-02T07:32:14.949450Z","iopub.status.idle":"2024-07-02T07:33:21.434478Z","shell.execute_reply.started":"2024-07-02T07:32:14.949425Z","shell.execute_reply":"2024-07-02T07:33:21.433375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_test_ds","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:21.438726Z","iopub.execute_input":"2024-07-02T07:33:21.439435Z","iopub.status.idle":"2024-07-02T07:33:21.446681Z","shell.execute_reply.started":"2024-07-02T07:33:21.439402Z","shell.execute_reply":"2024-07-02T07:33:21.445482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_train_ds.set_format('torch', columns=['pixel_values', 'label'])\npreprocessed_val_ds.set_format('torch', columns=['pixel_values', 'label'])\npreprocessed_test_ds.set_format('torch', columns=['pixel_values', 'label'])","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:21.448141Z","iopub.execute_input":"2024-07-02T07:33:21.448480Z","iopub.status.idle":"2024-07-02T07:33:21.606113Z","shell.execute_reply.started":"2024-07-02T07:33:21.448451Z","shell.execute_reply":"2024-07-02T07:33:21.605071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_test_sub.set_format('torch', columns=['pixel_values', 'label'])","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:21.607578Z","iopub.execute_input":"2024-07-02T07:33:21.608127Z","iopub.status.idle":"2024-07-02T07:33:21.616439Z","shell.execute_reply.started":"2024-07-02T07:33:21.608086Z","shell.execute_reply":"2024-07-02T07:33:21.615480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_train_ds","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:21.617710Z","iopub.execute_input":"2024-07-02T07:33:21.618038Z","iopub.status.idle":"2024-07-02T07:33:21.627681Z","shell.execute_reply.started":"2024-07-02T07:33:21.618010Z","shell.execute_reply":"2024-07-02T07:33:21.626587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# create dataloaders\ntrain_batch_size = 256\neval_batch_size = 256\ntrain_dataloader = torch.utils.data.DataLoader(preprocessed_train_ds, batch_size=train_batch_size, shuffle=True, num_workers=2)\nval_dataloader = torch.utils.data.DataLoader(preprocessed_val_ds, batch_size=eval_batch_size, num_workers=2)\ntest_dataloader = torch.utils.data.DataLoader(preprocessed_test_ds, batch_size=eval_batch_size, num_workers=2)\nbatch = next(iter(train_dataloader))","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:21.629086Z","iopub.execute_input":"2024-07-02T07:33:21.629429Z","iopub.status.idle":"2024-07-02T07:33:25.634423Z","shell.execute_reply.started":"2024-07-02T07:33:21.629401Z","shell.execute_reply":"2024-07-02T07:33:25.632801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader_sub = torch.utils.data.DataLoader(preprocessed_test_sub, batch_size=eval_batch_size, num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:25.636265Z","iopub.execute_input":"2024-07-02T07:33:25.636638Z","iopub.status.idle":"2024-07-02T07:33:25.642731Z","shell.execute_reply.started":"2024-07-02T07:33:25.636602Z","shell.execute_reply":"2024-07-02T07:33:25.641521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit_model = ViTModel.from_pretrained('google/vit-base-patch16-224-in21k')\nfor param in vit_model.parameters():\n      param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:25.644131Z","iopub.execute_input":"2024-07-02T07:33:25.644536Z","iopub.status.idle":"2024-07-02T07:33:27.904671Z","shell.execute_reply.started":"2024-07-02T07:33:25.644499Z","shell.execute_reply":"2024-07-02T07:33:27.903639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViTForImageClassification(nn.Module):\n    def __init__(self, num_labels=10):\n        super(ViTForImageClassification, self).__init__()\n#         self.config = config\n#         self.embedding = Embeddings(config)\n        self.vit = vit_model\n        self.dropout = nn.Dropout(0.1)\n        self.classifier = nn.Linear(self.vit.config.hidden_size, num_labels)\n        self.num_labels = num_labels\n\n    def forward(self, pixel_values):\n#         pixel_values = self.embedding(pixel_values)\n        outputs = self.vit(pixel_values=pixel_values)\n        output = self.dropout(outputs.last_hidden_state[:,0])\n        logits = self.classifier(output)\n        return logits","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:31.845551Z","iopub.execute_input":"2024-07-02T07:33:31.845944Z","iopub.status.idle":"2024-07-02T07:33:31.854119Z","shell.execute_reply.started":"2024-07-02T07:33:31.845913Z","shell.execute_reply":"2024-07-02T07:33:31.852856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ViTForImageClassification()\nmodel = model.to(device)\nsummary(model)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:32.960547Z","iopub.execute_input":"2024-07-02T07:33:32.961515Z","iopub.status.idle":"2024-07-02T07:33:33.332153Z","shell.execute_reply.started":"2024-07-02T07:33:32.961477Z","shell.execute_reply":"2024-07-02T07:33:33.331040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = AdamW(model.parameters(), lr = 1e-2, weight_decay=1e-4)\nepochs = 50","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:39.442149Z","iopub.execute_input":"2024-07-02T07:33:39.442926Z","iopub.status.idle":"2024-07-02T07:33:39.455402Z","shell.execute_reply.started":"2024-07-02T07:33:39.442888Z","shell.execute_reply":"2024-07-02T07:33:39.454095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train():  \n    model.train()\n    total_loss = 0\n    total_preds=[]\n    total_losses=[]\n\n    for step,batch in tqdm(enumerate(train_dataloader), total = len(train_dataloader)):\n        if step % 256 == 0 and not step == 0:\n            print(f'Batch {step} of {len(train_dataloader)}.')\n        \n        lbl, pix = batch.items()\n        lbl, pix = lbl[1].to(device), pix[1].to(device)\n        \n        preds = model(pix)\n        loss = torch.nn.functional.cross_entropy(preds, lbl)\n        total_losses\n        total_loss = total_loss + loss.item()\n        loss.backward()\n        \n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        optimizer.step()\n        optimizer.zero_grad()  \n        preds=preds.detach().cpu().numpy()\n        \n        total_preds.append(preds)\n        \n    avg_loss = total_loss / len(train_dataloader)\n  \n    total_preds  = np.concatenate(total_preds, axis=0)\n    return avg_loss, total_preds","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:44.806479Z","iopub.execute_input":"2024-07-02T07:33:44.806880Z","iopub.status.idle":"2024-07-02T07:33:44.818292Z","shell.execute_reply.started":"2024-07-02T07:33:44.806848Z","shell.execute_reply":"2024-07-02T07:33:44.817190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval():\n    total_loss = 0\n    model.eval()\n    for step,batch in tqdm(enumerate(val_dataloader), total=len(val_dataloader)):\n        lbl, pix = batch.items()\n        lbl, pix = lbl[1].to(device), pix[1].to(device)\n        \n        preds = model(pix)\n        loss = torch.nn.functional.cross_entropy(preds, lbl)\n        total_loss += loss.item()\n    \n    return total_loss / len(val_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:49.633502Z","iopub.execute_input":"2024-07-02T07:33:49.633907Z","iopub.status.idle":"2024-07-02T07:33:49.641313Z","shell.execute_reply.started":"2024-07-02T07:33:49.633870Z","shell.execute_reply":"2024-07-02T07:33:49.640275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_loss = inf\nes = 0\nfor epoch in range(epochs):\n    print(f'\\n Epoch {epoch+1} / {epochs}')\n    train_loss, _ = train()\n    val_loss = eval()\n    \n    if val_loss < min_loss:\n        min_loss = val_loss\n        es = 0\n    else:\n        es += 1\n        if es > 4:\n            print(\"Early stopping with train_loss: \", train_loss, \"and val_loss for this epoch: \", val_loss, \"...\")\n            break\n    \n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    print(f'\\n Training Loss: {train_loss}')\n    print(f'\\n Validation Loss: {val_loss}')","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:33:59.926358Z","iopub.execute_input":"2024-07-02T07:33:59.926932Z","iopub.status.idle":"2024-07-02T07:55:19.852597Z","shell.execute_reply.started":"2024-07-02T07:33:59.926881Z","shell.execute_reply":"2024-07-02T07:55:19.850515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval():\n    model.eval()\n    y_pred = []\n    y_true = []\n    with torch.no_grad():\n        for step, batch in tqdm(enumerate(test_dataloader), total = len(test_dataloader)):\n            lbl, pix = batch.items()\n            lbl, pix = lbl[1].to(device), pix[1].to(device)\n\n            outputs = model(pix)\n            outputs = torch.argmax(outputs, axis=1)\n            y_pred.extend(outputs.cpu().detach().numpy())\n            y_true.extend(lbl.cpu().detach().numpy())\n            \n    return y_pred, y_true\n\ny_pred, y_true = eval()","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:55:27.475140Z","iopub.execute_input":"2024-07-02T07:55:27.476074Z","iopub.status.idle":"2024-07-02T07:56:15.866176Z","shell.execute_reply.started":"2024-07-02T07:55:27.476030Z","shell.execute_reply":"2024-07-02T07:56:15.864794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit data\ndef eval_sub():\n    model.eval()\n    y_pred = []\n    y_true = []\n    with torch.no_grad():\n        for step, batch in tqdm(enumerate(test_dataloader_sub), total = len(test_dataloader_sub)):\n            lbl, pix = batch.items()\n            lbl, pix = lbl[1].to(device), pix[1].to(device)\n\n            outputs = model(pix)\n            outputs = torch.argmax(outputs, axis=1)\n            y_pred.extend(outputs.cpu().detach().numpy())\n            y_true.extend(lbl.cpu().detach().numpy())\n            \n    return y_pred, y_true","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:56:17.631250Z","iopub.execute_input":"2024-07-02T07:56:17.632053Z","iopub.status.idle":"2024-07-02T07:56:17.640964Z","shell.execute_reply.started":"2024-07-02T07:56:17.632009Z","shell.execute_reply":"2024-07-02T07:56:17.639745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct = np.array(y_pred) == np.array(y_true)\naccuracy = correct.sum() / len(correct)\nprint(\"Accuracy of the model\", accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:56:20.614769Z","iopub.execute_input":"2024-07-02T07:56:20.615466Z","iopub.status.idle":"2024-07-02T07:56:20.622759Z","shell.execute_reply.started":"2024-07-02T07:56:20.615430Z","shell.execute_reply":"2024-07-02T07:56:20.621476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nlabels = train_ds.features['label'].names\ncm = confusion_matrix(y_true, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\ndisp.plot(xticks_rotation=45)","metadata":{"execution":{"iopub.status.busy":"2024-07-01T08:48:55.040630Z","iopub.execute_input":"2024-07-01T08:48:55.041292Z","iopub.status.idle":"2024-07-01T08:48:55.855721Z","shell.execute_reply.started":"2024-07-01T08:48:55.041260Z","shell.execute_reply":"2024-07-01T08:48:55.854740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_sub, y_true_sub = eval_sub()","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:56:26.487703Z","iopub.execute_input":"2024-07-02T07:56:26.488086Z","iopub.status.idle":"2024-07-02T07:58:23.945888Z","shell.execute_reply.started":"2024-07-02T07:56:26.488057Z","shell.execute_reply":"2024-07-02T07:58:23.944590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct = np.array(y_pred_sub) == np.array(y_true_sub)\naccuracy = correct.sum() / len(correct)\nprint(\"Accuracy of the model\", accuracy)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:27.815935Z","iopub.execute_input":"2024-07-02T07:58:27.816385Z","iopub.status.idle":"2024-07-02T07:58:27.824718Z","shell.execute_reply.started":"2024-07-02T07:58:27.816344Z","shell.execute_reply":"2024-07-02T07:58:27.823730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_ds.features['label'].names\nlabels","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:30.099298Z","iopub.execute_input":"2024-07-02T07:58:30.099724Z","iopub.status.idle":"2024-07-02T07:58:30.107303Z","shell.execute_reply.started":"2024-07-02T07:58:30.099690Z","shell.execute_reply":"2024-07-02T07:58:30.106174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# max(y_pred_sub)\ny_pred_cat = [labels[i] for i in y_pred_sub]\ny_pred_cat[:10]","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:33.187847Z","iopub.execute_input":"2024-07-02T07:58:33.188217Z","iopub.status.idle":"2024-07-02T07:58:33.197995Z","shell.execute_reply.started":"2024-07-02T07:58:33.188188Z","shell.execute_reply":"2024-07-02T07:58:33.196821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nlst = [i+1 for i in range(10000)]\ndf_sub = pd.DataFrame(list(zip(lst, y_pred_cat)), columns =['id', 'label'])\ndf_sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:37.604011Z","iopub.execute_input":"2024-07-02T07:58:37.604410Z","iopub.status.idle":"2024-07-02T07:58:37.658812Z","shell.execute_reply.started":"2024-07-02T07:58:37.604377Z","shell.execute_reply":"2024-07-02T07:58:37.658007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/working/submission.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:38.570933Z","iopub.execute_input":"2024-07-02T07:58:38.571617Z","iopub.status.idle":"2024-07-02T07:58:38.616919Z","shell.execute_reply.started":"2024-07-02T07:58:38.571584Z","shell.execute_reply":"2024-07-02T07:58:38.615899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nlabels = train_ds.features['label'].names\ncm = confusion_matrix(y_true, y_pred)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\ndisp.plot(xticks_rotation=45)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:46.845910Z","iopub.execute_input":"2024-07-02T07:58:46.846295Z","iopub.status.idle":"2024-07-02T07:58:47.762055Z","shell.execute_reply.started":"2024-07-02T07:58:46.846266Z","shell.execute_reply":"2024-07-02T07:58:47.760958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_test_ds","metadata":{"execution":{"iopub.status.busy":"2024-07-01T12:29:50.134087Z","iopub.execute_input":"2024-07-01T12:29:50.134567Z","iopub.status.idle":"2024-07-01T12:29:50.142107Z","shell.execute_reply.started":"2024-07-01T12:29:50.134520Z","shell.execute_reply":"2024-07-01T12:29:50.141184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import py7zr\nfrom PIL import Image\nimport os\n\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n    z.extractall('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2024-07-02T07:58:57.232278Z","iopub.execute_input":"2024-07-02T07:58:57.232690Z","iopub.status.idle":"2024-07-02T08:01:35.930226Z","shell.execute_reply.started":"2024-07-02T07:58:57.232650Z","shell.execute_reply":"2024-07-02T08:01:35.929204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_filenames = [os.path.join('/kaggle/working/test', f) for f in os.listdir('/kaggle/working/test')]\ntest_filenames.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:01:52.198058Z","iopub.execute_input":"2024-07-02T08:01:52.198427Z","iopub.status.idle":"2024-07-02T08:01:53.381032Z","shell.execute_reply.started":"2024-07-02T08:01:52.198390Z","shell.execute_reply":"2024-07-02T08:01:53.380192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport numpy as np\n\nclass ExternalTestDataset(Dataset):\n    def __init__(self, filenames, preprocess_fn):\n        self.filenames = filenames\n        self.preprocess_fn = preprocess_fn\n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx]).convert(\"RGB\")\n        example = {'img': img}\n        processed_example = self.preprocess_fn(example)\n        return processed_example['pixel_values']\n\n# Define the preprocess function to be used in the dataset class\ndef preprocess_images_single(example):\n    images = [np.array(example['img'], dtype=np.uint8)]\n    images = [np.moveaxis(image, source=-1, destination=0) for image in images]\n    inputs = feature_extractor(images=images)\n    example['pixel_values'] = inputs['pixel_values'][0]\n    return example","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:01:54.419804Z","iopub.execute_input":"2024-07-02T08:01:54.420533Z","iopub.status.idle":"2024-07-02T08:01:54.430079Z","shell.execute_reply.started":"2024-07-02T08:01:54.420496Z","shell.execute_reply":"2024-07-02T08:01:54.428906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = ExternalTestDataset(test_filenames, preprocess_images_single)\ntest_loader = DataLoader(test_dataset, batch_size=256, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:01:56.313778Z","iopub.execute_input":"2024-07-02T08:01:56.314179Z","iopub.status.idle":"2024-07-02T08:01:56.319624Z","shell.execute_reply.started":"2024-07-02T08:01:56.314150Z","shell.execute_reply":"2024-07-02T08:01:56.318468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit data\ndef eval_sub():\n    model.eval()\n    y_pred = []\n    y_true = []\n    with torch.no_grad():\n        for step, batch in tqdm(enumerate(test_loader), total = len(test_loader)):\n            lbl, pix = batch.items()\n            lbl, pix = lbl[1].to(device), pix[1].to(device)\n\n            outputs = model(pix)\n            outputs = torch.argmax(outputs, axis=1)\n            y_pred.extend(outputs.cpu().detach().numpy())\n            y_true.extend(lbl.cpu().detach().numpy())\n            \n    return y_pred, y_true","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:02:52.131825Z","iopub.execute_input":"2024-07-02T08:02:52.132814Z","iopub.status.idle":"2024-07-02T08:02:52.140537Z","shell.execute_reply.started":"2024-07-02T08:02:52.132779Z","shell.execute_reply":"2024-07-02T08:02:52.139309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_sub, y_true_sub = eval_sub()","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:03:32.314797Z","iopub.execute_input":"2024-07-02T08:03:32.315216Z","iopub.status.idle":"2024-07-02T08:03:33.152697Z","shell.execute_reply.started":"2024-07-02T08:03:32.315184Z","shell.execute_reply":"2024-07-02T08:03:33.151374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct = np.array(y_pred_sub) == np.array(y_true_sub)\naccuracy = correct.sum() / len(correct)\nprint(\"Accuracy of the model\", accuracy)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nfrom transformers import ViTFeatureExtractor\n\n# Initialize the feature extractor\nfeature_extractor = ViTFeatureExtractor.from_pretrained('google/vit-base-patch16-224-in21k')\n\n# Directory containing external test images\nimage_dir = \"/kaggle/working/test\"\ntest_filenames = [os.path.join(image_dir, filename) for filename in os.listdir(image_dir) if filename.endswith(('png', 'jpg', 'jpeg'))]","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:12:12.092435Z","iopub.execute_input":"2024-07-02T08:12:12.092820Z","iopub.status.idle":"2024-07-02T08:12:12.998260Z","shell.execute_reply.started":"2024-07-02T08:12:12.092789Z","shell.execute_reply":"2024-07-02T08:12:12.997347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ExternalTestDataset(Dataset):\n    def __init__(self, filenames, preprocess_fn):\n        self.filenames = filenames\n        self.preprocess_fn = preprocess_fn\n\n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.filenames[idx]).convert(\"RGB\")\n        example = {'img': img}\n        processed_example = self.preprocess_fn(example)\n        return processed_example['pixel_values']\n\n# Preprocess function for single images\ndef preprocess_images_single(example):\n    image = np.array(example['img'], dtype=np.uint8)\n    image = np.moveaxis(image, source=-1, destination=0)\n    inputs = feature_extractor(images=[image])\n    example['pixel_values'] = inputs['pixel_values'][0]\n    return example","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:12:25.184296Z","iopub.execute_input":"2024-07-02T08:12:25.184689Z","iopub.status.idle":"2024-07-02T08:12:25.193150Z","shell.execute_reply.started":"2024-07-02T08:12:25.184656Z","shell.execute_reply":"2024-07-02T08:12:25.192044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-07-02T09:29:16.376867Z","iopub.execute_input":"2024-07-02T09:29:16.377535Z","iopub.status.idle":"2024-07-02T09:29:16.383634Z","shell.execute_reply.started":"2024-07-02T09:29:16.377503Z","shell.execute_reply":"2024-07-02T09:29:16.382679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the dataset\nexternal_test_dataset = ExternalTestDataset(test_filenames, preprocess_images_single)\n\n# Create DataLoader\neval_batch_size = 256\nexternal_test_dataloader = DataLoader(external_test_dataset, batch_size=eval_batch_size, shuffle=False, num_workers=2)\n\n# Verify the DataLoader\nbatch = next(iter(external_test_dataloader))\nprint(batch)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:12:39.112867Z","iopub.execute_input":"2024-07-02T08:12:39.113679Z","iopub.status.idle":"2024-07-02T08:12:41.310852Z","shell.execute_reply.started":"2024-07-02T08:12:39.113645Z","shell.execute_reply":"2024-07-02T08:12:41.309735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom tqdm import tqdm\n\ndef eval_sub():\n    model.eval()\n    y_pred = []\n    \n    with torch.no_grad():\n        for step, batch in tqdm(enumerate(external_test_dataloader), total=len(external_test_dataloader)):\n            batch = batch.to(device)\n            \n            outputs = model(batch)\n            outputs = torch.argmax(outputs, axis=1)\n            y_pred.extend(outputs.cpu().detach().numpy())\n            \n    return y_pred","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:14:27.866139Z","iopub.execute_input":"2024-07-02T08:14:27.867179Z","iopub.status.idle":"2024-07-02T08:14:27.876763Z","shell.execute_reply.started":"2024-07-02T08:14:27.867120Z","shell.execute_reply":"2024-07-02T08:14:27.875486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform evaluation\npredictions = eval_sub()\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T08:14:31.876429Z","iopub.execute_input":"2024-07-02T08:14:31.877179Z","iopub.status.idle":"2024-07-02T09:12:21.190209Z","shell.execute_reply.started":"2024-07-02T08:14:31.877141Z","shell.execute_reply":"2024-07-02T09:12:21.189071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_cat = [labels[i] for i in predictions]\ny_pred_cat[:10]","metadata":{"execution":{"iopub.status.busy":"2024-07-02T09:15:18.656731Z","iopub.execute_input":"2024-07-02T09:15:18.657719Z","iopub.status.idle":"2024-07-02T09:15:18.698596Z","shell.execute_reply.started":"2024-07-02T09:15:18.657678Z","shell.execute_reply":"2024-07-02T09:15:18.697492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_filenames[4224][21:-4]\nimg_id = [int(i[21:-4]) for i in test_filenames]\nimg_id[0]","metadata":{"execution":{"iopub.status.busy":"2024-07-02T09:30:31.432126Z","iopub.execute_input":"2024-07-02T09:30:31.432539Z","iopub.status.idle":"2024-07-02T09:30:31.532801Z","shell.execute_reply.started":"2024-07-02T09:30:31.432506Z","shell.execute_reply":"2024-07-02T09:30:31.531746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ndf_sub = pd.DataFrame(list(zip(img_id, y_pred_cat)), columns =['id', 'label'])\ndf_sub.to_csv('submission_final.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-02T09:30:53.459284Z","iopub.execute_input":"2024-07-02T09:30:53.459688Z","iopub.status.idle":"2024-07-02T09:30:54.107175Z","shell.execute_reply.started":"2024-07-02T09:30:53.459658Z","shell.execute_reply":"2024-07-02T09:30:54.106058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/working/submission_final.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2024-07-02T09:30:55.878735Z","iopub.execute_input":"2024-07-02T09:30:55.879639Z","iopub.status.idle":"2024-07-02T09:30:55.979667Z","shell.execute_reply.started":"2024-07-02T09:30:55.879594Z","shell.execute_reply":"2024-07-02T09:30:55.978542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset\ndata = [{'image': img} for img in test_images]\ntest_images_dataset = Dataset.from_list(data)\ntest_images_dataset.set_format(type='numpy', columns=['image'])\nprint(type(test_images_dataset))","metadata":{"execution":{"iopub.status.busy":"2024-07-01T13:14:36.317931Z","iopub.execute_input":"2024-07-01T13:14:36.318917Z","iopub.status.idle":"2024-07-01T13:14:39.058708Z","shell.execute_reply.started":"2024-07-01T13:14:36.318877Z","shell.execute_reply":"2024-07-01T13:14:39.057672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}