{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport random\nimport numpy as np\n%matplotlib inline\nfrom PIL import Image # Install Pillow -> conda install anaconda::pillow or pip install pillow\nimport os\nfrom skimage.io import  imread, imshow # Install scikit-image -> conda install scikit-image or pip install scikit-image\nimport torch\nimport torch.nn as nn\nimport re\nimport torch.optim as optim\nfrom torchvision import datasets, transforms, models \nfrom torch.utils.data import Dataset, DataLoader, Subset, random_split\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, f1_score, accuracy_score, precision_score\nfrom sklearn import preprocessing\nimport pandas as pd\nimport seaborn as sns\nfrom imblearn.over_sampling import RandomOverSampler\nfrom sklearn.model_selection import GridSearchCV\nfrom torch.functional import F","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check a better way to train the model instead of the CPU\nif torch.backends.mps.is_available():\n    device = torch.device(\"mps\") # new macbooks\nelif torch.cuda.is_available():    \n    device = torch.device(\"cuda\") # NVIDIA CUDA\nelse:\n    device = torch.device(\"cpu\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using manual seed to ensure that results are reproducible\ntorch.manual_seed(42) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions","metadata":{}},{"cell_type":"code","source":"# accuracy is weak for unbalanced datasets\n# overfitting you are remembering the cases of the training data instead of learning from it\ndef evaluate_network(net, X, y, to_device=True): \n    # Set the model to evaluation mode\n    net.eval()\n    if to_device:\n        X = X.to(device)\n        net = net.to(device)\n\n    # Run the model on the test data\n    with torch.no_grad():\n        outputs = net(X)\n        _, predicted = torch.max(outputs.data, 1)\n\n    # Convert tensors to numpy arrays\n    if to_device:\n        predicted = predicted.to(\"cpu\")\n    predicted_np = predicted.numpy()\n    test_target_np = y.numpy()\n\n    # Compute confusion matrix and F1 score\n    conf_mat = confusion_matrix(test_target_np, predicted_np)\n    f1 = f1_score(test_target_np, predicted_np, average='weighted')\n    accuracy = accuracy_score(test_target_np, predicted_np)\n    precision = precision_score(test_target_np, predicted_np, average='weighted')\n\n    print('Confusion Matrix:\\n', conf_mat)\n    print('F1 Score: ', f1)\n    print('Accuracy: ', accuracy)\n    print('Precison: ', precision)\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(X_train, y_train, nn, criterion, optimizer, n_epochs, to_device=True, batch_size=32):\n    #send everything to the device (ideally a GPU)\n    if to_device:\n        nn = nn.to(device)\n        X_train = X_train.to(device)\n        y_train = y_train.to(device)\n\n    # Train the network\n    loss_values = []\n    acc_values = []\n    for epoch in range(n_epochs):\n        accu_loss = 0\n        acc = 0\n        n = 0\n        for i in range(0, X_train.size(0), batch_size): # the objective of the use of batch is to always improve the classifier in some sets of data\n            X_batch = X_train[i:i+batch_size]\n            y_batch = y_train[i:i+batch_size]\n\n            # Forward pass\n            outputs = nn(X_batch)\n            loss = criterion(outputs, y_batch)\n            \n            _, predicted = torch.max(outputs.data, 1)\n            \n            accuracy = accuracy_score(y_batch.cpu().detach().numpy(), predicted.cpu().detach().numpy())\n            \n            accu_loss += loss.item()\n            acc += accuracy   \n            \n            \n            # Backward and optimize - to improve the weights\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            \n            n+=1\n        \n        #if (epoch+1) % 10 == 0:\n        print ('Epoch [{}/{}], Loss: {:.4f}, Acc: {:.4f}'.format(epoch+1, n_epochs, accu_loss/n, acc/n))\n        loss_values.append(accu_loss/n)\n        acc_values.append(acc/n)\n        \n    return loss_values, acc_values, nn.to(\"cpu\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_file(model, x, filename, to_device=True):\n  # Predict Kaggle results\n  model.eval()\n  if to_device:\n        x = x.to(device)\n        model = model.to(device)\n        \n  with torch.no_grad():\n    outputs = model(x)\n    _, predicted = torch.max(outputs.data, 1) # gets the result with the highest probability\n    \n  # Convert tensors to numpy arrays\n  if to_device:\n      predicted = predicted.to(\"cpu\")\n\n  # Save output\n  df=pd.DataFrame({'ImageId': np.arange(x.size(0))+1, 'Label': predicted})\n  df.to_csv(filename, index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_acc_function(loss_values, acc_values,labels): \n    fig, axes = plt.subplots(1, 2, figsize=(10, 5)) # Create subplots\n    \n    axes[0].plot(loss_values, label='Training Loss')\n    axes[0].set_xlabel('Epoch')\n    axes[0].set_ylabel('Loss')\n    axes[0].set_title('Loss Values')\n    \n    axes[1].plot(acc_values, label='Training Acc')\n    axes[1].set_xlabel('Epoch')\n    axes[1].set_ylabel('Loss')\n    axes[1].set_title('Acc Value')\n    \n    plt.suptitle(labels)\n    plt.tight_layout()\n    plt.legend(loc=\"lower right\")\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Pre-processing ","metadata":{}},{"cell_type":"code","source":"# Load the data\nx_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nx_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ny_train = x_train.pop('label')\n\nx_train = x_train.to_numpy()\ny_train = y_train.to_numpy()\nx_test = x_test.to_numpy()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking for missing values #\nprint(np.isnan(x_train).any())\nprint(np.isnan(x_test).any())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot some images\nplt.figure(figsize=(7, 7))\nfor i in range(16):\n    ax = plt.subplot(4, 4, i + 1)\n    plt.imshow(x_train[i].reshape(28, 28), cmap='gray')\n    plt.title(y_train[i])\n    plt.axis(\"off\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#binarize the data - black pixels are 1 and white pixels are 0\nx_train = np.where(x_train>0.5,1,0)\nx_test = np.where(x_test>0.5,1,0)\n\nplt.figure(figsize=(7, 7))\nfor i in range(16):\n    ax = plt.subplot(4, 4, i + 1)\n    plt.imshow(x_train[i].reshape(28, 28), cmap='gray')\n    plt.title(y_train[i])\n    plt.axis(\"off\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the distribution of each number\ndf = pd.DataFrame(list(pd.Series(y_train).value_counts().items()), columns=['Label', 'Count'])\nplt.bar(df['Label'], df['Count'])\nplt.xlabel('Labels')\nplt.ylabel('Counts')\nplt.title('Count of Labels')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Oversampling\nros = RandomOverSampler(random_state=42) # Instantiate the RandomOversampler\nx_train_, y_train_ = ros.fit_resample(x_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the distribution of each number\ndf = pd.DataFrame(list(pd.Series(y_train_).value_counts().items()), columns=['Label', 'Count'])\nplt.bar(df['Label'], df['Count'])\nplt.xlabel('Labels')\nplt.ylabel('Counts')\nplt.title('Count of Labels')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert into tensors\nx_train_t = torch.tensor(x_train_).float()\ny_train_t = torch.tensor(y_train_).long()\n\nx_test_t = torch.tensor(x_test).float()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_t = x_train_t.view(-1, 1, 28, 28)\nx_test_t = x_test_t.view(-1, 1, 28, 28)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"markdown","source":"##### **ii - Simple CNN**","metadata":{}},{"cell_type":"markdown","source":"###### **Architecture**","metadata":{}},{"cell_type":"code","source":"# Shallow CNN\nclass CNN_simple(nn.Module):\n    def __init__(self, input_channels=3, num_classes=9):\n        super(CNN_simple, self).__init__()\n        self.conv1 = nn.Conv2d(input_channels, 32, kernel_size=3, padding=1) # will extract 16 filters from the input\n        self.conv2 = nn.Conv2d(32, 16, kernel_size=3, padding = 1) # will extract 8 filters from the output of the other convolution layer\n        \n        #It consists of 8 filters of size 18x18 (due to pooling)\n        self.fc1 = nn.Linear(16 * 7 * 7, 32)\n        self.fc2 = nn.Linear(32, num_classes)\n\n    def forward(self, x):  \n        x = F.max_pool2d(F.relu(self.conv1(x)), 2) \n        x =  F.max_pool2d(F.relu(self.conv2(x)), 2)\n        x = x.view(-1, self.fc1.in_features)\n        \n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)\n        \n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 30\nBATCH_SIZE = 30\ncnn = CNN_simple(1, 10)\noptimizer = optim.SGD(cnn.parameters(), lr=0.01)\ncriterion = nn.CrossEntropyLoss() \n\nloss_values, acc_values, cnn = fit(x_train_t, y_train_t, cnn, criterion, optimizer, n_epochs, batch_size=BATCH_SIZE, to_device=False)\nplot_loss_acc_function(loss_values, acc_values, 'CNN')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_file(cnn,x_test_t, 'submission.csv')","metadata":{},"execution_count":null,"outputs":[]}]}