{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport os\nimport platform\nimport sys\n\n# linear algebra\nimport numpy as np\n# data processing, CSV file I/O (e.g. pd.read_csv)\nimport pandas as pd\n# plot\nimport matplotlib\nimport matplotlib.pyplot as plt\n# machine learning\nimport sklearn\n# deep learning\nimport torch\nimport torchvision","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-27T23:19:57.450745Z","iopub.execute_input":"2023-12-27T23:19:57.451140Z","iopub.status.idle":"2023-12-27T23:20:03.613325Z","shell.execute_reply.started":"2023-12-27T23:19:57.451104Z","shell.execute_reply":"2023-12-27T23:20:03.612115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Operating system: \", platform.platform(), platform.version())\nprint(\"Python version\", sys.version)\nprint(\"Pandas version: \", pd.__version__)\nprint(\"Sklearn version: \", sklearn.__version__)\nprint(\"Pytorch version: \", torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.616222Z","iopub.execute_input":"2023-12-27T23:20:03.617059Z","iopub.status.idle":"2023-12-27T23:20:03.626950Z","shell.execute_reply.started":"2023-12-27T23:20:03.616999Z","shell.execute_reply":"2023-12-27T23:20:03.625286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device = torch.device(\"cuda\")\n    print(torch.cuda.get_device_name(0))\n    print('Memory Usage:')\n    print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')\n    print('Cached:   ', round(torch.cuda.memory_reserved(0)/1024**3,1), 'GB')\nelif torch.backends.mps.is_available() and torch.backends.mps.is_built():\n    device = torch.device(\"mps\")\nelse:\n    device = torch.device(\"cpu\")\n\nprint('Using device:', device)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.628391Z","iopub.execute_input":"2023-12-27T23:20:03.629474Z","iopub.status.idle":"2023-12-27T23:20:03.641167Z","shell.execute_reply.started":"2023-12-27T23:20:03.629434Z","shell.execute_reply":"2023-12-27T23:20:03.639826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_STATE = 0\nTRAIN_VS_VALIDATION_SET_PERCENTANGE = 0.8\nBATCH_SIZE = 32\nTRAIN_FULL_IMAGES_PATH = \"/kaggle/temp/train\"\nTEST_FULL_IMAGES_PATH = \"/kaggle/temp/test\"\nTRAIN_IMAGES_REORGANIZED_PATH = \"/kaggle/temp/train_reorganized\"\nVALID_IMAGES_REORGANIZED_PATH = \"/kaggle/temp/valid_reorganized\"\nTEST_IMAGES_REORGANIZED_PATH = \"/kaggle/temp/test_reorganized\"\nTEST_IMAGES_REORGANIZED_PLACEHOLDER_LABEL_PATH = f\"{TEST_IMAGES_REORGANIZED_PATH}/unknown\"","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.643687Z","iopub.execute_input":"2023-12-27T23:20:03.644105Z","iopub.status.idle":"2023-12-27T23:20:03.653180Z","shell.execute_reply.started":"2023-12-27T23:20:03.644054Z","shell.execute_reply":"2023-12-27T23:20:03.651675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\ntrain_full_labels = pd.read_csv('../input/cifar-10/trainLabels.csv', index_col='id')","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.655781Z","iopub.execute_input":"2023-12-27T23:20:03.656698Z","iopub.status.idle":"2023-12-27T23:20:03.720724Z","shell.execute_reply.started":"2023-12-27T23:20:03.656654Z","shell.execute_reply":"2023-12-27T23:20:03.719209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Break off validation set from training data\ntrain_labels, valid_labels = train_test_split(\n    train_full_labels,\n    train_size=TRAIN_VS_VALIDATION_SET_PERCENTANGE,\n    # shuffle the dataset to ensure the traning set is representive of the overall distribution of the data,\n    # this would be useful e.g. the input is sorted by target label or specific feature.\n    shuffle=True,\n    random_state=RANDOM_STATE\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.722102Z","iopub.execute_input":"2023-12-27T23:20:03.722658Z","iopub.status.idle":"2023-12-27T23:20:03.900134Z","shell.execute_reply.started":"2023-12-27T23:20:03.722618Z","shell.execute_reply":"2023-12-27T23:20:03.898666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.902167Z","iopub.execute_input":"2023-12-27T23:20:03.902643Z","iopub.status.idle":"2023-12-27T23:20:03.923737Z","shell.execute_reply.started":"2023-12-27T23:20:03.902605Z","shell.execute_reply":"2023-12-27T23:20:03.922460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr --quiet","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:03.925700Z","iopub.execute_input":"2023-12-27T23:20:03.926111Z","iopub.status.idle":"2023-12-27T23:20:22.968247Z","shell.execute_reply.started":"2023-12-27T23:20:03.926075Z","shell.execute_reply":"2023-12-27T23:20:22.967031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# You can calso write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n!ls \"$TRAIN_FULL_IMAGES_PATH\" > /dev/null && echo \"train set has been already extracted\" || python -m py7zr x \"../input/cifar-10/train.7z\" \"/kaggle/temp/\"\n!ls \"$TEST_FULL_IMAGES_PATH\" > /dev/null && echo \"test set has been already extracted\" || python -m py7zr x \"../input/cifar-10/test.7z\" \"/kaggle/temp/\"","metadata":{"execution":{"iopub.status.busy":"2023-12-27T23:20:22.970394Z","iopub.execute_input":"2023-12-27T23:20:22.971130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://d2l.ai/chapter_computer-vision/kaggle-cifar10.html#reading-the-dataset\ndef reorganize_images(labels, source_path, destination_path):\n    for label in train_labels[\"label\"].unique():\n        class_path = f\"{destination_path}/{label}\"\n        print(f\"creating class path {class_path}\")\n        os.makedirs(class_path, exist_ok=True)\n    for idx, row in labels.iterrows():\n        id = row.name\n        label = row[\"label\"]\n        destination_image_path = f\"{destination_path}/{label}/{id}.png\"\n        if os.path.isfile(destination_image_path):\n            os.remove(destination_image_path)\n        os.symlink(f\"{source_path}/{id}.png\", destination_image_path)\n\nreorganize_images(train_labels, TRAIN_FULL_IMAGES_PATH, TRAIN_IMAGES_REORGANIZED_PATH)\nreorganize_images(valid_labels, TRAIN_FULL_IMAGES_PATH, VALID_IMAGES_REORGANIZED_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#load-and-normalize-cifar10\nfrom torchvision.transforms import Compose, ToTensor, Normalize\n\nmean = torch.tensor([0.4914, 0.4822, 0.4465])\nstd = torch.tensor([0.2023, 0.1994, 0.2010])\n\n# Image data preprocessing pineline\n# Composes several transforms together.\ntransform = Compose([\n    # The output of torchvision datasets are PILImage images of range [0, 1]\n    # Convert a PIL Image or numpy.ndarray to tensor\n    ToTensor(),\n    # # We transform them to Tensors of normalized range [-1, 1].\n    # Normalize a tensor image with mean and standard deviation\n    # input[channel] = (input[channel] - mean[channel]) / std[channel]\n    # https://towardsdatascience.com/understand-data-normalization-in-machine-learning-8ff3062101f0\n    # Standardization improves the numerical stability of your model\n    Normalize(mean, std)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. Load and normalizing the CIFAR10 training and test datasets using torchvision\nfrom torchvision.datasets import ImageFolder\n\ntrain_images = ImageFolder(TRAIN_IMAGES_REORGANIZED_PATH, transform=transform)\nvalid_images = ImageFolder(VALID_IMAGES_REORGANIZED_PATH, transform=transform)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_img(img):\n    # Permute the dimensions of an array\n    # Put the two image axes in the front\n    PIL_RGB_image = img.permute(1, 2, 0)\n    # unnormalize\n    img = PIL_RGB_image * std + mean\n    plt.imshow(img.numpy())\n\nsample_image, sample_label_index = train_images[0]\ndisplay_img(sample_image)\nprint(train_images.classes[sample_label_index])\nprint(sample_image.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2. Define a Convolution Neural Network\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(3, 6, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n        self.conv2 = nn.Conv2d(6, 16, 5)\n        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 10)\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = x.view(-1, 16 * 5 * 5)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\n\nnet = Net()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3. Define a Loss function and optimizer\nimport torch.optim as optim\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is an iterative process that gradually try to decrease the loss term. Due to the performance and stability consideration we use small set of samples at one time to update the parameters. We call this batch size, total number of training examples present in a sinple improvement. And we call one epoch when an entire dataset is passed forward and backward through the neural network once. And the number of iteration requiered to pass the whole epoch is calculate as total number of tranning example divided by batch size.","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import DataLoader\n\n# Create train and validation batch for training\ntrain_loader = DataLoader(dataset=train_images, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nvalid_loader = DataLoader(dataset=valid_images, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 4. Train the network on the training data\nepochs = 2\nfor epoch in range(epochs):  # loop over the dataset multiple times\n    running_loss = 0.0\n    for batch_index, batch in enumerate(train_loader):\n        # get the inputs\n        sample_images, sample_labels = batch\n\n        # zero the parameter gradients\n        # by default PyTorch accumulates the gradients on subsequent backward passes\n        optimizer.zero_grad()\n\n        # forward + backward + optimize\n        predicted_labels = net(sample_images)\n        loss = criterion(predicted_labels, sample_labels)\n        loss.backward()\n        optimizer.step()\n\n        # print statistics\n        running_loss += loss.item()\n        # print every 500 mini-batches\n        if batch_index % 500 == 0:\n            print(f\"epoch {epoch} batch {batch_index} loss {running_loss / 2000}\")\n        running_loss = 0\n\n    loss = criterion(predicted_labels, sample_labels)\n    train_loss = loss.item()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p \"$TEST_IMAGES_REORGANIZED_PATH\"\n!ls \"$TEST_IMAGES_REORGANIZED_PLACEHOLDER_LABEL_PATH\" > /dev/null && echo \"test set has been already linked\" || ln -s \"$TEST_FULL_IMAGES_PATH\" \"$TEST_IMAGES_REORGANIZED_PLACEHOLDER_LABEL_PATH\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images = ImageFolder(TEST_IMAGES_REORGANIZED_PATH, transform=transform)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Subset\nfrom torchvision.utils import make_grid\n\n# get some test images\nsamples_test_images = Subset(test_images, range(5))\nsample_images, ignored_labeles_indeces = zip(*samples_test_images)\n# show sample test images\ndisplay_img(make_grid(list(sample_images)))\n# Okay, now let us see what the neural network thinks these examples above are\nfor sample_image in sample_images:\n    pred = net(sample_image)\n    print(train_images.classes[pred.argmax()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = DataLoader(dataset=test_images, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\npreds = []\nfor X, _ in test_loader:\n    y_hat = net(X)\n    preds.extend(y_hat.argmax(dim=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\nids = list(map(lambda image_path: image_path[len(TEST_IMAGES_REORGANIZED_PLACEHOLDER_LABEL_PATH) + 1:-4], map(lambda image_info: image_info[0][0], zip(test_images.imgs))))\ndf = pd.DataFrame({'id': ids, 'label': map(lambda i: train_images.classes[i], preds)})\ndf.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}