{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Table of Content:\n1. [Import Required Libraries](#1)\n2. [Load and Prepare Dataset](#2)\n3. [Build RNN Model](#3)\n4. [Create Model Instance](#4)\n5. [Define Loss Function](#5)\n6. [Set Up Optimizer](#6)\n7. [Train the Model](#7)\n8. [Make Predictions](#7)\n9. [Visualization Loss & Accuracy](#8) ","metadata":{}},{"cell_type":"markdown","source":"<a id=\"1\"></a> \n# Import Required Libraries","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:30.347239Z","iopub.execute_input":"2025-03-04T19:02:30.347598Z","iopub.status.idle":"2025-03-04T19:02:30.355738Z","shell.execute_reply.started":"2025-03-04T19:02:30.347559Z","shell.execute_reply":"2025-03-04T19:02:30.354479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, TensorDataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:30.357131Z","iopub.execute_input":"2025-03-04T19:02:30.357666Z","iopub.status.idle":"2025-03-04T19:02:33.825471Z","shell.execute_reply.started":"2025-03-04T19:02:30.357596Z","shell.execute_reply":"2025-03-04T19:02:33.824499Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"2\"></a> \n# Load and Prepare Dataset","metadata":{}},{"cell_type":"markdown","source":"## Load Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(r\"/kaggle/input/digit-recognizer/train.csv\", dtype=np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:33.827294Z","iopub.execute_input":"2025-03-04T19:02:33.827761Z","iopub.status.idle":"2025-03-04T19:02:36.754360Z","shell.execute_reply.started":"2025-03-04T19:02:33.827733Z","shell.execute_reply":"2025-03-04T19:02:36.753539Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prepare Dataset","metadata":{}},{"cell_type":"code","source":"targets_numpy = train.label.values\nfeatures_numpy = train.loc[:,train.columns != \"label\"].values/255\n#features_numpy = features_numpy.values / 255","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:36.756083Z","iopub.execute_input":"2025-03-04T19:02:36.756599Z","iopub.status.idle":"2025-03-04T19:02:36.851593Z","shell.execute_reply.started":"2025-03-04T19:02:36.756557Z","shell.execute_reply":"2025-03-04T19:02:36.850465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_numpy.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:36.852636Z","iopub.execute_input":"2025-03-04T19:02:36.853028Z","iopub.status.idle":"2025-03-04T19:02:36.860083Z","shell.execute_reply.started":"2025-03-04T19:02:36.852997Z","shell.execute_reply":"2025-03-04T19:02:36.858960Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"targets_numpy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:36.861063Z","iopub.execute_input":"2025-03-04T19:02:36.861422Z","iopub.status.idle":"2025-03-04T19:02:36.879085Z","shell.execute_reply.started":"2025-03-04T19:02:36.861395Z","shell.execute_reply":"2025-03-04T19:02:36.877942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_train, features_test, targets_train, targets_test = train_test_split(features_numpy,targets_numpy,test_size=0.2,random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:36.880225Z","iopub.execute_input":"2025-03-04T19:02:36.880587Z","iopub.status.idle":"2025-03-04T19:02:37.316370Z","shell.execute_reply.started":"2025-03-04T19:02:36.880559Z","shell.execute_reply":"2025-03-04T19:02:37.315388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"features_train shape:\",features_train.shape)\nprint(\"targets_train shape:\",targets_train.shape)\nprint(\"features_test shape:\",features_test.shape)\nprint(\"targets_test shape:\",targets_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.317316Z","iopub.execute_input":"2025-03-04T19:02:37.317691Z","iopub.status.idle":"2025-03-04T19:02:37.324413Z","shell.execute_reply.started":"2025-03-04T19:02:37.317662Z","shell.execute_reply":"2025-03-04T19:02:37.322813Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create Tensors","metadata":{}},{"cell_type":"code","source":"print(features_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.325481Z","iopub.execute_input":"2025-03-04T19:02:37.325954Z","iopub.status.idle":"2025-03-04T19:02:37.341861Z","shell.execute_reply.started":"2025-03-04T19:02:37.325913Z","shell.execute_reply":"2025-03-04T19:02:37.340629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresTrain = torch.from_numpy(features_train)\ntargetsTrain = torch.from_numpy(targets_train).type(torch.LongTensor)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.345957Z","iopub.execute_input":"2025-03-04T19:02:37.346335Z","iopub.status.idle":"2025-03-04T19:02:37.361419Z","shell.execute_reply.started":"2025-03-04T19:02:37.346298Z","shell.execute_reply":"2025-03-04T19:02:37.360177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(featuresTrain.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.363465Z","iopub.execute_input":"2025-03-04T19:02:37.363878Z","iopub.status.idle":"2025-03-04T19:02:37.381827Z","shell.execute_reply.started":"2025-03-04T19:02:37.363845Z","shell.execute_reply":"2025-03-04T19:02:37.380692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresTest = torch.from_numpy(features_test)\ntargetsTest = torch.from_numpy(targets_test).type(torch.LongTensor)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.382993Z","iopub.execute_input":"2025-03-04T19:02:37.383393Z","iopub.status.idle":"2025-03-04T19:02:37.398993Z","shell.execute_reply.started":"2025-03-04T19:02:37.383357Z","shell.execute_reply":"2025-03-04T19:02:37.397897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch_size = 100\nn_iters = 8000\nnum_epochs = n_iters / (len(targets_train) / batch_size)\nnum_epochs = int(num_epochs)\n\nprint(\"number of epochs:\",num_epochs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.400074Z","iopub.execute_input":"2025-03-04T19:02:37.400477Z","iopub.status.idle":"2025-03-04T19:02:37.415943Z","shell.execute_reply.started":"2025-03-04T19:02:37.400434Z","shell.execute_reply":"2025-03-04T19:02:37.414954Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### TensorDataset \nCombines one or more torch.Tensor objects together.","metadata":{}},{"cell_type":"code","source":"print(featuresTrain.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.417024Z","iopub.execute_input":"2025-03-04T19:02:37.417347Z","iopub.status.idle":"2025-03-04T19:02:37.438025Z","shell.execute_reply.started":"2025-03-04T19:02:37.417314Z","shell.execute_reply":"2025-03-04T19:02:37.436921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = TensorDataset(featuresTrain,targetsTrain)\ntest = TensorDataset(featuresTest,targetsTest)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.439059Z","iopub.execute_input":"2025-03-04T19:02:37.439488Z","iopub.status.idle":"2025-03-04T19:02:37.456455Z","shell.execute_reply.started":"2025-03-04T19:02:37.439451Z","shell.execute_reply":"2025-03-04T19:02:37.455373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_samples = featuresTrain.shape[0]  \nprint(f\"Num Samples: {num_samples}\")\nprint(featuresTrain.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.457619Z","iopub.execute_input":"2025-03-04T19:02:37.458017Z","iopub.status.idle":"2025-03-04T19:02:37.474786Z","shell.execute_reply.started":"2025-03-04T19:02:37.457980Z","shell.execute_reply":"2025-03-04T19:02:37.473696Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### DataLoader\nIn PyTorch, when training a deep learning model, you need to feed data into your model. However, instead of loading the data directly into your model, the DataLoader class splits the data into small chunks (`batch_size`) and feeds these chunks to your model sequentially.","metadata":{}},{"cell_type":"code","source":"train_loader = DataLoader(train, batch_size = batch_size, shuffle = False)\ntest_loader = DataLoader(test, batch_size = batch_size, shuffle = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.476012Z","iopub.execute_input":"2025-03-04T19:02:37.476409Z","iopub.status.idle":"2025-03-04T19:02:37.494153Z","shell.execute_reply.started":"2025-03-04T19:02:37.476374Z","shell.execute_reply":"2025-03-04T19:02:37.493146Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"3\"></a> \n# Build RNN Model","metadata":{"execution":{"iopub.status.busy":"2025-03-04T14:52:23.360800Z","iopub.execute_input":"2025-03-04T14:52:23.361133Z","iopub.status.idle":"2025-03-04T14:52:23.365338Z","shell.execute_reply.started":"2025-03-04T14:52:23.361109Z","shell.execute_reply":"2025-03-04T14:52:23.364262Z"}}},{"cell_type":"code","source":"class RNNModel(nn.Module):\n    def __init__(self, input_dimension,hidden_dimension,layer_dimension,output_dimension):\n        super(RNNModel,self).__init__()\n        \n        self.hidden_dimension = hidden_dimension\n        \n        self.layer_dimension = layer_dimension\n\n        self.rnn = nn.RNN(input_dimension,hidden_dimension,layer_dimension,batch_first=True,nonlinearity=\"relu\")\n\n        # full connected layer\n        self.fc = nn.Linear(hidden_dimension,output_dimension)\n\n    def forward(self,x):\n        # x->img\n        \n        hidden_state = torch.zeros(self.layer_dimension, x.size(0),self.hidden_dimension)\n\n        output, _ = self.rnn(x,hidden_state)\n        output =  self.fc(output[:, -1, :])\n        # output = (batch_size, time_steps, hidden_dimension)\n        return output\n\n# The final hidden state of the RNN passes through a fully connected layer and produces the model's prediction (output)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.495203Z","iopub.execute_input":"2025-03-04T19:02:37.495607Z","iopub.status.idle":"2025-03-04T19:02:37.509809Z","shell.execute_reply.started":"2025-03-04T19:02:37.495567Z","shell.execute_reply":"2025-03-04T19:02:37.508644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch_size = 100\nn_iters = 8000\nnum_epochs = n_iters / (len(features_train) / batch_size)\nnum_epochs = int(num_epochs)\nprint(\"number of epochs:\",num_epochs)\n\n# Pytorch train and test sets\ntrain = TensorDataset(featuresTrain,targetsTrain)\ntest = TensorDataset(featuresTest,targetsTest)\n\n# data loader\ntrain_loader = DataLoader(train, batch_size = batch_size, shuffle = False)\ntest_loader = DataLoader(test, batch_size = batch_size, shuffle = False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.511101Z","iopub.execute_input":"2025-03-04T19:02:37.511564Z","iopub.status.idle":"2025-03-04T19:02:37.528936Z","shell.execute_reply.started":"2025-03-04T19:02:37.511521Z","shell.execute_reply":"2025-03-04T19:02:37.527944Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"4\"></a> \n# Create Model Instance","metadata":{}},{"cell_type":"code","source":"input_dim = 28    # input dimension\nhidden_dim = 100  # hidden layer dimension\nlayer_dim = 1     # number of hidden layers\noutput_dim = 10   # output dimension\n\nmodel = RNNModel(input_dim, hidden_dim, layer_dim, output_dim)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.530082Z","iopub.execute_input":"2025-03-04T19:02:37.530525Z","iopub.status.idle":"2025-03-04T19:02:37.547834Z","shell.execute_reply.started":"2025-03-04T19:02:37.530485Z","shell.execute_reply":"2025-03-04T19:02:37.546778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"5\"></a> \n# Define Loss Function\n\n### Cross Entropy Loss ","metadata":{}},{"cell_type":"code","source":"error = nn.CrossEntropyLoss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.549109Z","iopub.execute_input":"2025-03-04T19:02:37.549538Z","iopub.status.idle":"2025-03-04T19:02:37.569035Z","shell.execute_reply.started":"2025-03-04T19:02:37.549499Z","shell.execute_reply":"2025-03-04T19:02:37.567872Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"6\"></a> \n# Set Up Optimizer\n\n### SGD Optimizer","metadata":{}},{"cell_type":"code","source":"learning_rate = 0.02\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:37.569863Z","iopub.execute_input":"2025-03-04T19:02:37.570161Z","iopub.status.idle":"2025-03-04T19:02:38.628427Z","shell.execute_reply.started":"2025-03-04T19:02:37.570134Z","shell.execute_reply":"2025-03-04T19:02:38.627624Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"7\"></a> \n# Train the Model & Make Predictions\n\n### `outputs.data`\n- `outputs` are the predictions generated by the model for each sample, and they are usually a **logits** (raw scores) matrix. The size of this matrix is `(batch_size, num_classes)`, meaning it contains the raw scores for each class for each sample.\n\n### `torch.max(outputs.data, 1)`\n- The `torch.max(input, dim)` function returns the maximum value and its index along the specified `dim` of the `input` tensor.\n- `dim=1` here is used to find the class with the highest score for each sample. With `dim=1`, it selects the maximum value and its index along each row (which corresponds to each sample).\n- This works as follows:\n  - `outputs` tensor will be of size `batch_size x num_classes`. For example, if there are 1000 samples and 10 classes, the size will be `(1000, 10)`.\n  - `torch.max(outputs, 1)` returns the highest value and its index for each sample (row). In this case, we want to get the index of the class with the highest score.\n\n### `[1]`\n- `torch.max(outputs.data, 1)` returns two things: the maximum values (`max_values`) and their corresponding indices (`indices`).\n- The function `outputs.data.max(1)` returns a pair:\n  - `max_values`: The highest score for each sample.\n  - `indices`: The index of the class with the highest score for each sample (class label).\n- The `[1]` is used here to select only the **indices**, which represent the predicted class for each sample.\n\n### Result\n- The `predicted` tensor will contain the indices of the class labels for each sample.\n","metadata":{}},{"cell_type":"code","source":"seq_dim = 28  \nloss_list = []\niteration_list = []\naccuracy_list = []\ncount = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:38.629324Z","iopub.execute_input":"2025-03-04T19:02:38.629756Z","iopub.status.idle":"2025-03-04T19:02:38.634286Z","shell.execute_reply.started":"2025-03-04T19:02:38.629730Z","shell.execute_reply":"2025-03-04T19:02:38.633168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(train_loader):\n        train  = images.view(-1, seq_dim, input_dim)\n        labels = labels.long()\n            \n        # Clear gradients\n        optimizer.zero_grad()\n        \n        # Forward propagation\n        outputs = model(train)\n        \n        # Calculate softmax and ross entropy loss\n        loss = error(outputs, labels)\n        \n        # Calculating gradients\n        loss.backward()\n        \n        # Update parameters\n        optimizer.step()\n        \n        count += 1\n        \n        if count % 250 == 0:\n            # Calculate Accuracy         \n            correct = 0\n            total = 0\n            # Iterate through test dataset\n            for images, labels in test_loader:\n                images = images.view(-1, seq_dim, input_dim)\n                \n                # Forward propagation\n                outputs = model(images)\n                \n                # Get predictions from the maximum value\n                predicted = torch.max(outputs.data,dim=1)[1]\n                \n                # Total number of labels\n                total += labels.size(0)\n                \n                correct += (predicted == labels).sum()\n            \n            accuracy = 100 * correct / float(total)\n            \n            # store loss and iteration\n            loss_list.append(loss.data)\n            iteration_list.append(count)\n            accuracy_list.append(accuracy)\n            if count % 500 == 0:\n                print('Iteration: {}  Loss: {}  Accuracy: {} %'.format(count, loss.item(), accuracy))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:02:38.635527Z","iopub.execute_input":"2025-03-04T19:02:38.635898Z","iopub.status.idle":"2025-03-04T19:04:04.326857Z","shell.execute_reply.started":"2025-03-04T19:02:38.635864Z","shell.execute_reply":"2025-03-04T19:04:04.325722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id=\"8\"></a> \n# Visualization Loss & Accuracy","metadata":{}},{"cell_type":"code","source":"# visualization loss \nplt.plot(iteration_list,loss_list)\nplt.xlabel(\"Number of iteration\")\nplt.ylabel(\"Loss\")\nplt.title(\"RNN: Loss vs Number of iteration\")\nplt.show()\n\n# visualization accuracy \nplt.plot(iteration_list,accuracy_list,color = \"red\")\nplt.xlabel(\"Number of iteration\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"RNN: Accuracy vs Number of iteration\")\nplt.savefig('graph.png')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-04T19:04:04.327923Z","iopub.execute_input":"2025-03-04T19:04:04.328236Z","iopub.status.idle":"2025-03-04T19:04:04.797749Z","shell.execute_reply.started":"2025-03-04T19:04:04.328199Z","shell.execute_reply":"2025-03-04T19:04:04.796434Z"}},"outputs":[],"execution_count":null}]}