{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Very simple custom CSS style.\n\nfrom IPython.core.display import HTML\n\ncustom_css = \"\"\"\n<style>\nh1,h2,h3,h4 {\ncolor:blue !important;\ntext-align: left !important;\npadding: 0px !important;\nborder-bottom: 2px solid blue !important;\n}\n</style>\"\"\"\n\nHTML(custom_css)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-06-13T02:33:21.835974Z","iopub.execute_input":"2024-06-13T02:33:21.836662Z","iopub.status.idle":"2024-06-13T02:33:21.842760Z","shell.execute_reply.started":"2024-06-13T02:33:21.836629Z","shell.execute_reply":"2024-06-13T02:33:21.841848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport timm\nfrom torch.utils.data import DataLoader, TensorDataset\nimport torch.nn as nn\nimport torch.optim as optim","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:00:55.405370Z","iopub.execute_input":"2024-06-11T04:00:55.405755Z","iopub.status.idle":"2024-06-11T04:00:55.411818Z","shell.execute_reply.started":"2024-06-11T04:00:55.405722Z","shell.execute_reply":"2024-06-11T04:00:55.410739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data Files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nsample = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\n\nimage_size = np.sqrt(len(train.columns[1:]))\nprint(f\"image size = {image_size} x {image_size}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:00:58.322038Z","iopub.execute_input":"2024-06-11T04:00:58.322374Z","iopub.status.idle":"2024-06-11T04:01:02.543977Z","shell.execute_reply.started":"2024-06-11T04:00:58.322349Z","shell.execute_reply":"2024-06-11T04:01:02.543058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test for Cuda","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:05.590545Z","iopub.execute_input":"2024-06-11T04:01:05.591427Z","iopub.status.idle":"2024-06-11T04:01:05.596845Z","shell.execute_reply.started":"2024-06-11T04:01:05.591392Z","shell.execute_reply":"2024-06-11T04:01:05.595859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Input Dataframes","metadata":{}},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:10.129461Z","iopub.execute_input":"2024-06-11T04:01:10.130125Z","iopub.status.idle":"2024-06-11T04:01:10.146406Z","shell.execute_reply.started":"2024-06-11T04:01:10.130089Z","shell.execute_reply":"2024-06-11T04:01:10.145538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:10.977446Z","iopub.execute_input":"2024-06-11T04:01:10.978255Z","iopub.status.idle":"2024-06-11T04:01:10.994278Z","shell.execute_reply.started":"2024-06-11T04:01:10.978221Z","shell.execute_reply":"2024-06-11T04:01:10.993228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:11.574370Z","iopub.execute_input":"2024-06-11T04:01:11.574736Z","iopub.status.idle":"2024-06-11T04:01:11.583324Z","shell.execute_reply.started":"2024-06-11T04:01:11.574705Z","shell.execute_reply":"2024-06-11T04:01:11.582412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Sampling of Images","metadata":{}},{"cell_type":"code","source":"# Set number of rows, columns.\nN = 28\n\n# Randomly choose 100 rows to display\nrow_list = random.sample(range(784), 100)\n\n# Display row values as images\nfig = plt.figure(figsize=(12, 14))\n\nfig.patch.set_facecolor(\"lightblue\")\nfig.suptitle(\"Some Sample Digits from Training Set\", fontsize=28)\nfig.subplots_adjust(top=0.93)\n\nfor i, row in enumerate(row_list):\n    pic = train.drop(columns=\"label\").iloc[row].values\n    pic = pic.reshape((N, N))\n    label = train.iloc[row][\"label\"]\n    ax = fig.add_subplot(10, 10, i + 1)\n    ax.imshow(pic, cmap=\"gray\")\n    ax.set_title(label, fontsize=14, fontweight=\"normal\")\n    ax.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:13.169997Z","iopub.execute_input":"2024-06-11T04:01:13.170360Z","iopub.status.idle":"2024-06-11T04:01:26.192235Z","shell.execute_reply.started":"2024-06-11T04:01:13.170330Z","shell.execute_reply":"2024-06-11T04:01:26.191314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Count of Each Digit in Training Dataset","metadata":{}},{"cell_type":"code","source":"digit_counts = train.label.value_counts().reindex(range(10))\nplt.style.use(\"bmh\")\nplt.figure(figsize=(10, 6))\nbars = plt.bar(digit_counts.index, digit_counts.values, color=\"coral\")\n\n# Put count on tops of bar\nfor bar in bars:\n    value = bar.get_height()\n    plt.text(\n        bar.get_x() + bar.get_width() / 2,\n        value + 0.05,\n        int(value),\n        ha=\"center\",\n        va=\"bottom\",\n    )\n\nplt.xlabel(\"Digit\")\nplt.ylabel(\"Count\")\nplt.title(\"Count of each digit in training dataset\", loc=\"left\")\nplt.xticks(digit_counts.index)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:31.741127Z","iopub.execute_input":"2024-06-11T04:01:31.741492Z","iopub.status.idle":"2024-06-11T04:01:32.056485Z","shell.execute_reply.started":"2024-06-11T04:01:31.741462Z","shell.execute_reply":"2024-06-11T04:01:32.055426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TIMM Model: resnet_18\n\nhttps://huggingface.co/gpcarl123/resnet18_mnist","metadata":{}},{"cell_type":"code","source":"%%time\n# Import libraries\nimport timm\nimport torchvision\nimport torch\nfrom torch.utils import data\nfrom torchvision import transforms\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n# Set random seeds\nseed = 42\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\nnp.random.seed(seed)\nrandom.seed(seed)\n\n# Set data paths\nTRAIN_CSV_PATH = \"/kaggle/input/digit-recognizer/train.csv\"\nTEST_CSV_PATH = \"/kaggle/input/digit-recognizer/test.csv\"\n\n# Load train and test data\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\ntest_df = pd.read_csv(TEST_CSV_PATH)\n\n\n# Define the Dataset Class\nclass DigitDataset(data.Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n        self.labels = df[\"label\"].values\n        self.images = (\n            df.drop(\"label\", axis=1).values.reshape(-1, 28, 28).astype(\"float32\")\n        )\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image = self.images[idx]\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n\n# Split the training data into train and validation sets\ntrain_df, val_df = train_test_split(train_df, test_size=0.2, random_state=seed)\n\n# Define the transformations\ntransform = transforms.Compose(\n    [\n        transforms.ToPILImage(),\n        # transforms.Resize((32, 32)),  # Resizing to match ResNet input size\n        transforms.Resize((224, 224)),  # Resizing to match ResNet input size\n        transforms.ToTensor(),\n        torchvision.transforms.Normalize((0.1307,), (0.3081,)),\n    ]\n)\n\n# Create the torch datasets\ntrain_dataset = DigitDataset(train_df, transform=transform)\nval_dataset = DigitDataset(val_df, transform=transform)\n\n# Create DataLoaders\ntrain_loader = data.DataLoader(\n    train_dataset, batch_size=64, shuffle=True, num_workers=4\n)\nval_loader = data.DataLoader(val_dataset, batch_size=64, shuffle=False, num_workers=4)\n\n# Create the model\nnet = timm.create_model(\"resnet18\", pretrained=False, num_classes=10)\n\nnet.conv1 = torch.nn.Conv2d(\n    1, 64, kernel_size=7, stride=2, padding=3, bias=False\n)  # Adjusting the first layer for single-channel input\n\n# Move the model to the GPU if available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nnet = net.to(device)\n\n\n# Training Function\ndef train(model, train_loader, criterion, optimizer):\n    model.train()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n\n    for images, labels in train_loader:\n        images, labels = images.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n        _, predicted = torch.max(outputs, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\n    return running_loss / len(train_loader), correct / total\n\n\n# Validation Function\ndef validate(model, val_loader, criterion):\n    model.eval()\n    val_loss = 0.0\n    correct = 0\n    total = 0\n\n    # Lists for capturing images, labels, and predictions for misclassifications.\n    misclassified_images = []\n    misclassified_labels = []\n    misclassified_preds = []\n\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.to(device), labels.to(device)\n\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n            # Store misclassified digits\n            misclassified_indices = (predicted != labels).nonzero(as_tuple=True)[0]\n            misclassified_images.extend(images[misclassified_indices].cpu())\n            misclassified_labels.extend(labels[misclassified_indices].cpu())\n            misclassified_preds.extend(predicted[misclassified_indices].cpu())\n\n    return val_loss/len(val_loader), correct/total, misclassified_images, misclassified_labels, misclassified_preds\n\n\n# Training loop\nnum_epochs = 15\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(net.parameters(), lr=0.001)\n\nfor epoch in range(num_epochs):\n    train_loss, train_acc = train(net, train_loader, criterion, optimizer)\n    val_loss, val_acc, misclassified_images, misclassified_labels, misclassified_preds = validate(net, val_loader, criterion)\n\n    print(f\"Epoch {epoch+1}/{num_epochs}\")\n    print(f\"Train Loss: {train_loss:.4f}, Train Accuracy: {train_acc:.4f}\")\n    print(f\"Val Loss: {val_loss:.4f}, Val Accuracy: {val_acc:.4f}\")\n\n# Note the misclassified images,labels and predicitons are only used from the last epoch.\n\n\n# Evaluation on the test set\nclass TestDataset(data.Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n        self.images = df.values.reshape(-1, 28, 28).astype(\"float32\")\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image = self.images[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image\n\n\ntest_dataset = TestDataset(test_df, transform=transform)\ntest_loader = data.DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=4)\n\nnet.eval()\ntest_predictions = []\n\nwith torch.no_grad():\n    for images in test_loader:\n        images = images.to(device)\n        outputs = net(images)\n        _, predicted = torch.max(outputs, 1)\n        test_predictions.extend(predicted.cpu().numpy())\n\n# Create submission file\nsubmission_df = pd.DataFrame(\n    {\"ImageId\": range(1, len(test_predictions) + 1), \"Label\": test_predictions}\n)\nsubmission_df.to_csv(\"submission.csv\", index=False)\nprint(\"Predictions saved to submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:01:36.626476Z","iopub.execute_input":"2024-06-11T04:01:36.626904Z","iopub.status.idle":"2024-06-11T04:16:35.099689Z","shell.execute_reply.started":"2024-06-11T04:01:36.626872Z","shell.execute_reply":"2024-06-11T04:16:35.098608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Count Number of Misclassified Digits in Validation","metadata":{}},{"cell_type":"code","source":"print( len(misclassified_images)==len(misclassified_labels)==len(misclassified_preds) )\n\nnum_missed = len(misclassified_images)\n\nprint(f\"Number of misclassified digits: {num_missed}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:16:43.281833Z","iopub.execute_input":"2024-06-11T04:16:43.282222Z","iopub.status.idle":"2024-06-11T04:16:43.288155Z","shell.execute_reply.started":"2024-06-11T04:16:43.282186Z","shell.execute_reply":"2024-06-11T04:16:43.287150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Misclassified Digits\nNote this displays the transformed versions of the missed digits.","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(12, 120))  # Capture the figure object\nfig.patch.set_facecolor(\"lightblue\")\nfig.suptitle(\"Misclassified Digits (transformed)\", fontsize=28)\n\n# Adjust subplots to make room for the overall title\nfig.subplots_adjust(top=0.95, hspace=0.4)\n\nfor i, (image, label, pred) in enumerate(zip(misclassified_images, misclassified_labels, misclassified_preds)):\n    image = image.squeeze().numpy()\n    ax = fig.add_subplot(49, 5, i + 1)\n    ax.imshow(image, cmap=\"gray\")\n    ax.set_title(f\"Predicted: {pred.item()}, Actual: {label.item()}\")\n    ax.axis(\"off\")\n\nplt.tight_layout(rect=[0, 0, 1, 0.98])\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-11T04:16:53.468226Z","iopub.execute_input":"2024-06-11T04:16:53.468579Z","iopub.status.idle":"2024-06-11T04:17:00.774025Z","shell.execute_reply.started":"2024-06-11T04:16:53.468550Z","shell.execute_reply":"2024-06-11T04:17:00.773139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The End","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}