{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Analyst dataset\n*check out some infor about dataset*","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:29:59.924062Z","iopub.execute_input":"2025-08-01T19:29:59.924335Z","iopub.status.idle":"2025-08-01T19:30:07.144047Z","shell.execute_reply.started":"2025-08-01T19:29:59.924308Z","shell.execute_reply":"2025-08-01T19:30:07.143347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# check some basic info\nprint(train.shape)\nprint(train.info())\nprint(train.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:30:07.144857Z","iopub.execute_input":"2025-08-01T19:30:07.145124Z","iopub.status.idle":"2025-08-01T19:30:08.935952Z","shell.execute_reply.started":"2025-08-01T19:30:07.145096Z","shell.execute_reply":"2025-08-01T19:30:08.935177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_digit(index, data, has_label=True):\n    if has_label:\n        img = data.iloc[index, 1:].values.reshape(28, 28)\n        label = data.iloc[index, 0]\n    else:\n        img = data.iloc[index].values.reshape(28, 28)\n        label = \"UNK\"\n\n    # To show the images here\n    plt.imshow(img)\n    plt.title(f\"Label: {label}\")\n    plt.axis('off')\n    plt.show()\n\n\nfor i in range(2):\n    show_digit(i, train, has_label=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:30:08.937980Z","iopub.execute_input":"2025-08-01T19:30:08.938218Z","iopub.status.idle":"2025-08-01T19:30:09.203867Z","shell.execute_reply.started":"2025-08-01T19:30:08.938197Z","shell.execute_reply":"2025-08-01T19:30:09.203235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Some basic info about dataset:\n- This is a handwritten digist from 0 to 9 have size (28x28), input describe by 784 columns have value in range [0,255]. \n- Goal predict the digit represented in each image.","metadata":{}},{"cell_type":"markdown","source":"## Prepare for test training \n\n*I will try using scikit-learn to train on this dataset first before go to imporvement later on using deeplearning:*\n\n- First I will split the dataset for training and for validation.\n- Then I will training with 3 models to check their accuracy.","metadata":{}},{"cell_type":"code","source":"## Prepare data for training process also I will split the data for validaion.\nfrom sklearn.model_selection import train_test_split\n\n\nX = train.drop(\"label\", axis=1) / 255.0 # normalize the data input vectors\ny = train[\"label\"]\n\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:30:09.204593Z","iopub.execute_input":"2025-08-01T19:30:09.204840Z","iopub.status.idle":"2025-08-01T19:30:10.251275Z","shell.execute_reply.started":"2025-08-01T19:30:09.204811Z","shell.execute_reply":"2025-08-01T19:30:10.250425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.metrics import accuracy_score\n\nrandom_forest = RandomForestClassifier(n_estimators=100)\nrandom_forest.fit(X_train, y_train)\n\npreds = random_forest.predict(X_val)\nprint(\"Validation Accuracy:\", accuracy_score(y_val, preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:30:10.252201Z","iopub.execute_input":"2025-08-01T19:30:10.252739Z","iopub.status.idle":"2025-08-01T19:30:34.335940Z","shell.execute_reply.started":"2025-08-01T19:30:10.252708Z","shell.execute_reply":"2025-08-01T19:30:34.334987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=1000, alpha=1e-4)\nmlp.fit(X_train, y_train)\n\npreds = mlp.predict(X_val)\nprint(\"Validation Accuracy:\", accuracy_score(y_val, preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:30:34.336841Z","iopub.execute_input":"2025-08-01T19:30:34.337156Z","iopub.status.idle":"2025-08-01T19:31:32.535956Z","shell.execute_reply.started":"2025-08-01T19:30:34.337128Z","shell.execute_reply":"2025-08-01T19:31:32.534933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier(n_neighbors=5)\nknn.fit(X_train, y_train)\n\npreds = knn.predict(X_val)\nprint(\"Validation Accuracy:\", accuracy_score(y_val, preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:31:32.536916Z","iopub.execute_input":"2025-08-01T19:31:32.537139Z","iopub.status.idle":"2025-08-01T19:31:40.008510Z","shell.execute_reply.started":"2025-08-01T19:31:32.537121Z","shell.execute_reply":"2025-08-01T19:31:40.007716Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Conclusion 1\n- We get a good result from 96% to 97% accuracy.\n- The model appears to learn well\n- It still not achive to the state of art.","metadata":{}},{"cell_type":"markdown","source":"## Improvemenmt\n\n### The idea:\nUsing CNN a famous model, which are well-known and effective models for image classification tasks.\n\n## CNN\n\nDiffer from Neuron network CNN does not connect all neurons instead it just look into a small area of data (image) once.\n\nFor Convolution blocks we will have\n\nConvolution:\n$$\nY(i, j) = \\sum_{u=0}^{k-1} \\sum_{v=0}^{k-1} K(u, v) \\cdot X(i + u, j + v)\n$$\n\nActiavate function: Often is ReLU max(0, x) <x are input>\n\nPooling:\n\n$$\nY(i, j) = \\max_{(u,v) \\in \\text{window}} X(i + u, j + v)\n$$\n\nI will use 3 fundamental operations to build a network\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torchvision import transforms\nimport matplotlib.pyplot as plt\n\ntrain_df = train\ntest_df = test\n\nclass DigitDataset(Dataset):\n    def __init__(self, df, is_test=False):\n        self.is_test = is_test\n        self.X = df.values if is_test else df.iloc[:, 1:].values\n        self.X = self.X.reshape(-1, 1, 28, 28).astype('float32') / 255.0\n        self.y = None if is_test else df.iloc[:, 0].values.astype('int64')\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        if self.is_test:\n            return torch.tensor(self.X[idx])\n        return torch.tensor(self.X[idx]), torch.tensor(self.y[idx])\n\nfull_dataset = DigitDataset(train_df)\ntrain_size = int(0.9 * len(full_dataset))\nval_size = len(full_dataset) - train_size\ntrain_dataset, val_dataset = random_split(full_dataset, [train_size, val_size])\n\ntest_dataset = DigitDataset(test_df, is_test=True)\n\n# DataLoader\ntrain_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=64)\ntest_loader = DataLoader(test_dataset, batch_size=64)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:31:40.009289Z","iopub.execute_input":"2025-08-01T19:31:40.009544Z","iopub.status.idle":"2025-08-01T19:31:48.294035Z","shell.execute_reply.started":"2025-08-01T19:31:40.009524Z","shell.execute_reply":"2025-08-01T19:31:48.293185Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**For submition result I will get the result of the best model not from the latest**\n```\nBecause during training, the model is updated the weight over multiple epochs to avoid overfitting and ensure better generalization and test performance\n```","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm import tqdm\n\n## Pattern 1 \n# class CNNModel(nn.Module):\n#     def __init__(self):\n#         super(CNNModel, self).__init__()\n\n#         self.conv1 = nn.Conv2d(1, 64, kernel_size=3)\n#         self.conv2 = nn.Conv2d(64, 64, kernel_size=3)   \n#         self.pool1 = nn.MaxPool2d(2)   \n#         self.bn1 = nn.BatchNorm2d(64)\n\n#         self.conv3 = nn.Conv2d(64, 128, kernel_size=3)\n#         self.conv4 = nn.Conv2d(128, 128, kernel_size=3) \n#         self.pool2 = nn.MaxPool2d(2)      \n#         self.bn2 = nn.BatchNorm2d(128)\n\n#         self.conv5 = nn.Conv2d(128, 256, kernel_size=3)\n#         self.pool3 = nn.MaxPool2d(2)\n#         self.bn3 = nn.BatchNorm2d(256)\n\n#         self.flatten = nn.Flatten()\n#         self.fc1 = nn.Linear(256, 512)\n#         self.fc2 = nn.Linear(512, 10)\n\n#     def forward(self, x):\n#         x = F.relu(self.conv1(x))\n#         x = F.relu(self.conv2(x)) \n#         x = self.pool1(x) \n#         x = self.bn1(x)\n\n#         x = F.relu(self.conv3(x))\n#         x = F.relu(self.conv4(x)) \n#         x = self.pool2(x)              \n#         x = self.bn2(x)\n\n#         x = F.relu(self.conv5(x))        \n#         x = self.pool3(x)                \n#         x = self.bn3(x)\n\n#         x = self.flatten(x)             \n#         x = F.relu(self.fc1(x))          \n#         x = self.fc2(x)                 \n#         return x\n\n## Pattern 2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=3) \n        self.bn1 = nn.BatchNorm2d(64)\n        self.conv2 = nn.Conv2d(64, 64, kernel_size=3) \n        self.bn2 = nn.BatchNorm2d(64)\n        self.pool1 = nn.MaxPool2d(2)\n\n        self.conv3 = nn.Conv2d(64, 128, kernel_size=3)\n        self.bn3 = nn.BatchNorm2d(128)\n        self.conv4 = nn.Conv2d(128, 128, kernel_size=3)\n        self.bn4 = nn.BatchNorm2d(128)\n        self.pool2 = nn.MaxPool2d(2)\n\n        self.conv5 = nn.Conv2d(128, 256, kernel_size=3)\n        self.bn5 = nn.BatchNorm2d(256)\n        self.pool3 = nn.MaxPool2d(2)\n\n        self.flatten = nn.Flatten()\n        self.fc1 = nn.Linear(256, 512)\n        self.fc2 = nn.Linear(512, 10)\n\n    def forward(self, x):\n        x = F.relu(self.bn1(self.conv1(x)))\n        x = F.relu(self.bn2(self.conv2(x))) \n        x = self.pool1(x)\n\n        x = F.relu(self.bn3(self.conv3(x)))\n        x = F.relu(self.bn4(self.conv4(x)))\n        x = self.pool2(x)\n\n        x = F.relu(self.bn5(self.conv5(x)))\n        x = self.pool3(x)\n\n        x = self.flatten(x)\n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = CNNModel().to(device)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\nnum_epochs = 100\n\nbest_val_loss = float('inf')\nbest_model_state = None\n\nfor epoch in tqdm(range(num_epochs)):\n    model.train()\n    running_loss = 0.0\n    for images, labels in train_loader:\n        images, labels = images.to(device), labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n\n    epoch_loss = running_loss / len(train_loader)\n    print(f\"Epoch [{epoch+1}/{num_epochs}], Train Loss: {epoch_loss:.5f}\")\n\n    # validationn\n    model.eval()\n    val_loss = 0.0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n    val_loss /= len(val_loader)\n\n    print(f\"Validation Loss: {val_loss:.5f}\")\n\n    # get best model\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        best_model_state = model.state_dict()\n        print(\"Best model found\")\n\nlatest_model_state = model.state_dict()\nprint(\"Latest model done\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-01T19:31:48.296208Z","iopub.execute_input":"2025-08-01T19:31:48.296651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## helper function to chcek result \n\ndef evaluate_model(model, dataloader, device):\n    model.eval()\n    correct = 0\n    total = 0\n\n    with torch.no_grad():\n        for images, labels in dataloader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    accuracy = 100 * correct / total\n    return accuracy\n\n\nmodel.load_state_dict(best_model_state)\nval_accuracy = evaluate_model(model, val_loader, device)\nprint(f\"Validation Accuracy: {val_accuracy:.2f}%\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## get predictions from tetst set\nmodel.eval()\npredictions = []\n\nwith torch.no_grad():\n    for images in test_loader:\n        images = images.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.cpu().numpy())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## save a submition\nimport pandas as pd\n\nsubmission = pd.DataFrame(\n    {\n        \"ImageId\": list(range(1, len(predictions) + 1)),\n        \"Label\": predictions\n    }\n)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Conclusion 2 \n\n\n1. I have built two different CNN architectures and trained both using the same number of epochs (100).\n2. The second architecture improved the accuracy by approximately 1% compared to the first one.\n   *In pattern, **batch normalization** is placed after the pooling layers. But in the seccond pattern I have applied after each convolutional layer and before the activation function, also I use **batch normalization** more than the first pattern which might help improving generalization than the first one*\n4. Both CNN models achieved higher accuracy than models trained using traditional machine learning algorithms from the scikit-learn library (~2%).\n5. I don't added the dropout function because I have already added the batch normalization to regularize the model.","metadata":{}}]}