{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a href=\"https://colab.research.google.com/github/denis-spe/Digit-Recognization/blob/v1/digit_recognization_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>","metadata":{"id":"view-in-github"}},{"cell_type":"markdown","source":"<h1>Digit Recognization With Pytorch</h1>","metadata":{"id":"RAq-VCxmh5Vw"}},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Import Necessary Libraries and packages</h2>","metadata":{"id":"OWj3GPQ1h5V9"}},{"cell_type":"code","source":"!pip install torchinfo\n","metadata":{"id":"902_XujGOl2D","outputId":"56f64b80-6742-48f0-d3cc-9eb8ae64e5ff","execution":{"iopub.status.busy":"2022-04-15T00:16:49.439161Z","iopub.execute_input":"2022-04-15T00:16:49.439732Z","iopub.status.idle":"2022-04-15T00:17:04.228983Z","shell.execute_reply.started":"2022-04-15T00:16:49.439623Z","shell.execute_reply":"2022-04-15T00:17:04.227431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nimport pandas as pd\nimport os, warnings, time\nfrom PIL import Image\nfrom typing import Dict\nfrom torchinfo import summary\nfrom torch import nn as nn\nimport matplotlib.pyplot as plt\nfrom torch import functional as f\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nprint(\"Pytorch Version: \", torch.__version__)","metadata":{"id":"zMt6hD8jh5V-","outputId":"66c5b59b-2d2d-428a-ef04-6ad616b23dab","execution":{"iopub.status.busy":"2022-04-15T00:17:04.231968Z","iopub.execute_input":"2022-04-15T00:17:04.233177Z","iopub.status.idle":"2022-04-15T00:17:06.480661Z","shell.execute_reply.started":"2022-04-15T00:17:04.233129Z","shell.execute_reply":"2022-04-15T00:17:06.479512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Configuring The NoteBook</h2>","metadata":{"id":"bAofDxrMh5WE"}},{"cell_type":"code","source":"# Setting the graph style\nplt.rc('figure', autolayout=True)\nplt.rc(\n    'axes', titleweight='bold', \n    titlesize=20, labelweight=700,\n    labelsize=13\n    )\nplt.rc('font', size=15)\n\ndef set_seeds(seed: int=0):\n    \"\"\"Sets the Seed into order to same result on every code run\"\"\"\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n\n# Call the set_seeds with default parameter\nset_seeds()\n\n# Turning the warnings\nwarnings.filterwarnings('ignore')\n\n# Use the Gpu if available either cpu will be used\ndevice = 'cuda' if torch.cuda.is_available() else \"cpu\"\nprint(\"Using %s\" % device)\n","metadata":{"id":"qS1Be65_h5WF","outputId":"1a09b136-3563-4f1e-cc89-7ce777aa25ab","execution":{"iopub.status.busy":"2022-04-15T00:17:06.482233Z","iopub.execute_input":"2022-04-15T00:17:06.482671Z","iopub.status.idle":"2022-04-15T00:17:06.565387Z","shell.execute_reply.started":"2022-04-15T00:17:06.482534Z","shell.execute_reply":"2022-04-15T00:17:06.564328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h3 style=\"font-size: 20px;\">Setting the variable to be used in model building and training</h3>","metadata":{"id":"VromiOaMh5WJ"}},{"cell_type":"code","source":"BATCH_SIZE=64\nEPOCHS = 3\nLEARNING_RATE = 1e-3","metadata":{"id":"F7oWDesPh5WL","execution":{"iopub.status.busy":"2022-04-15T00:17:06.568672Z","iopub.execute_input":"2022-04-15T00:17:06.569388Z","iopub.status.idle":"2022-04-15T00:17:06.574754Z","shell.execute_reply.started":"2022-04-15T00:17:06.569340Z","shell.execute_reply":"2022-04-15T00:17:06.573310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Above Data</h2>\n<p style=\"font-size: 15px;\">\nThe data files train.csv and test.csv contain gray-scale images of hand-drawn digits, from zero through nine.\n\nEach image is 28 pixels in height and 28 pixels in width, for a total of 784 pixels in total. Each pixel has a single pixel-value associated with it, indicating the lightness or darkness of that pixel, with higher numbers meaning darker. This pixel-value is an integer between 0 and 255, inclusive.\n\nThe training data set, (train.csv), has 785 columns. The first column, called \"label\", is the digit that was drawn by the user. The rest of the columns contain the pixel-values of the associated image.\n\nEach pixel column in the training set has a name like pixelx, where x is an integer between 0 and 783, inclusive. To locate this pixel on the image, suppose that we have decomposed x as x = i * 28 + j, where i and j are integers between 0 and 27, inclusive. Then pixelx is located on row i and column j of a 28 x 28 matrix, (indexing by zero).\n\nFor example, pixel31 indicates the pixel that is in the fourth column from the left, and the second row from the top, as in the ascii-diagram below.\n\nVisually, if we omit the \"pixel\" prefix, the pixels make up the image like this:\n<pre>\n000 001 002 003 ... 026 027\n028 029 030 031 ... 054 055\n056 057 058 059 ... 082 083\n|   |   |   |  ...  |   |\n728 729 730 731 ... 754 755\n756 757 758 759 ... 782 783\n</pre>\n</p>","metadata":{"id":"qC0c6ztfh5WN"}},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Load Csv File Into Pandas DataFrame</h2>","metadata":{"id":"MBpHjGIuh5WP"}},{"cell_type":"code","source":"try:\n    from google.colab import drive\n    # Connect the google drive\n    drive.mount('/gdrive')\n\n    # list the files from drive\n    folder_path = '../gdrive/MyDrive/Datasets/MNIST_data'\nexcept ImportError:\n    folder_path = '../input/digit-recognizer'\n\nos.listdir(folder_path)","metadata":{"id":"-fJ3Mmu3h_Co","outputId":"b279e06e-75c9-4caf-8624-738ff968cdcf","execution":{"iopub.status.busy":"2022-04-15T00:17:06.576640Z","iopub.execute_input":"2022-04-15T00:17:06.577261Z","iopub.status.idle":"2022-04-15T00:17:06.596065Z","shell.execute_reply.started":"2022-04-15T00:17:06.577216Z","shell.execute_reply":"2022-04-15T00:17:06.594882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Path\ntrain_data_path = folder_path + '/train.csv'\ntest_data_path = folder_path + '/test.csv'\n\n\n","metadata":{"id":"htANu1tch5WQ","execution":{"iopub.status.busy":"2022-04-15T00:17:06.597604Z","iopub.execute_input":"2022-04-15T00:17:06.598613Z","iopub.status.idle":"2022-04-15T00:17:06.604023Z","shell.execute_reply.started":"2022-04-15T00:17:06.598556Z","shell.execute_reply":"2022-04-15T00:17:06.602933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading dataset into pandas \ntrain_df = pd.read_csv(train_data_path)\ntest_df = pd.read_csv(test_data_path)","metadata":{"id":"MG7ObC8XixQ_","execution":{"iopub.status.busy":"2022-04-15T00:17:06.605639Z","iopub.execute_input":"2022-04-15T00:17:06.606931Z","iopub.status.idle":"2022-04-15T00:17:13.122878Z","shell.execute_reply.started":"2022-04-15T00:17:06.606870Z","shell.execute_reply":"2022-04-15T00:17:13.121867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">View The Dataframe</h2>","metadata":{"id":"6-sQevvSh5WS"}},{"cell_type":"code","source":"# First Rows in the train data\nprint(train_df.shape)\ntrain_df.head()","metadata":{"id":"IH8kp5lyh5WT","outputId":"85264bfd-14b7-4cf5-c6ca-d0ac3728474f","execution":{"iopub.status.busy":"2022-04-15T00:17:13.124267Z","iopub.execute_input":"2022-04-15T00:17:13.124596Z","iopub.status.idle":"2022-04-15T00:17:13.156022Z","shell.execute_reply.started":"2022-04-15T00:17:13.124546Z","shell.execute_reply":"2022-04-15T00:17:13.154722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First Rows in the test data\nprint(test_df.shape)\ntest_df.head()","metadata":{"id":"j01AA56Ih5WU","outputId":"01865d45-50ed-468f-dedd-ed44df6cca04","execution":{"iopub.status.busy":"2022-04-15T00:17:13.157697Z","iopub.execute_input":"2022-04-15T00:17:13.158282Z","iopub.status.idle":"2022-04-15T00:17:13.183601Z","shell.execute_reply.started":"2022-04-15T00:17:13.158234Z","shell.execute_reply":"2022-04-15T00:17:13.182450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Split Train Dataset Into Train And Validation Set</h2>","metadata":{"id":"7E0CD7qV5Szq"}},{"cell_type":"code","source":"mask = np.random.randn(len(train_df)) <= 0.8\ntrain_data = train_df[mask]\nvalid_data = train_df[~mask]\nprint(f\"Train_data Shape {train_data.shape}\\nValid_data: {valid_data.shape}\")","metadata":{"id":"yZOWFflx5G54","outputId":"dd80e18b-a927-407c-dd7e-54285ace0a24","execution":{"iopub.status.busy":"2022-04-15T00:17:13.188473Z","iopub.execute_input":"2022-04-15T00:17:13.188727Z","iopub.status.idle":"2022-04-15T00:17:13.341660Z","shell.execute_reply.started":"2022-04-15T00:17:13.188696Z","shell.execute_reply":"2022-04-15T00:17:13.340522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Creating Dataset</h2>","metadata":{"id":"G18wkTxbh5WV"}},{"cell_type":"code","source":"class CreateDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, transforms: transforms = None, \n                 train: bool =True, normalize: bool = True):\n        self.transforms = transforms\n        self.train = train\n        self.df = df\n        self.len = self.df.shape[0]\n        self.normalize = normalize\n\n    def __len__(self):\n        return self.len\n\n    def __getitem__(self, index: int): \n        if 'label' in self.df.columns:\n            image = self.df.iloc[index, 1:].values.reshape(28, 28)\n            if self.normalize:\n                image = self.df.iloc[index, 1:].values.reshape(28, 28) / 255\n            label = self.df.iloc[index, 0]\n            if self.transforms:\n                image = self.transforms(image).type(torch.float)\n            return image, label\n        else:\n            image = self.df.iloc[index, :].values.reshape(28, 28)\n            if self.normalize:\n                image = self.df.iloc[index, :].values.reshape(28, 28) / 255\n            if self.transforms:\n                image = self.transforms(image).type(torch.float)\n            return image","metadata":{"id":"vsS_Z0xih5WV","execution":{"iopub.status.busy":"2022-04-15T00:17:13.343132Z","iopub.execute_input":"2022-04-15T00:17:13.343661Z","iopub.status.idle":"2022-04-15T00:17:13.355909Z","shell.execute_reply.started":"2022-04-15T00:17:13.343623Z","shell.execute_reply":"2022-04-15T00:17:13.354678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Construct Datasets train_X_set \ntrain_X_set = CreateDataset(\n    train_data, \n    transforms=transforms.Compose([transforms.ToTensor()]))\n\n# Construct valid_X_set dataset\nvalid_X_set = CreateDataset(\n    valid_data, \n    transforms=transforms.Compose([transforms.ToTensor()]))\n\n# Construct test_data \ntest_data = CreateDataset(\n    test_df, \n    transforms=transforms.Compose([transforms.ToTensor()]))","metadata":{"id":"S2ogKeC-sG05","execution":{"iopub.status.busy":"2022-04-15T00:17:13.357535Z","iopub.execute_input":"2022-04-15T00:17:13.358095Z","iopub.status.idle":"2022-04-15T00:17:13.372181Z","shell.execute_reply.started":"2022-04-15T00:17:13.358050Z","shell.execute_reply":"2022-04-15T00:17:13.370883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Placing Dataset Into Dataloader</h2>","metadata":{"id":"_9bELv8oh5WW"}},{"cell_type":"code","source":"# Coupling Datasets with dataloader function\ntrain_dataloader = DataLoader(train_X_set, batch_size=BATCH_SIZE, shuffle=True)\nvalid_dataloader = DataLoader(valid_X_set, batch_size=BATCH_SIZE, shuffle=True)\n\ntest_dataloader = DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"id":"Ev82xBN7h5WX","execution":{"iopub.status.busy":"2022-04-15T00:17:13.374080Z","iopub.execute_input":"2022-04-15T00:17:13.374720Z","iopub.status.idle":"2022-04-15T00:17:13.383990Z","shell.execute_reply.started":"2022-04-15T00:17:13.374671Z","shell.execute_reply":"2022-04-15T00:17:13.383023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Exploring Datasets</h2>","metadata":{"id":"uockZt5th5WY"}},{"cell_type":"code","source":"# Getting the first image and label in the train_dataset\nimg, label = train_dataloader.dataset[0]","metadata":{"id":"tN7ZRYojh5WY","execution":{"iopub.status.busy":"2022-04-15T00:17:13.385908Z","iopub.execute_input":"2022-04-15T00:17:13.386341Z","iopub.status.idle":"2022-04-15T00:17:13.411225Z","shell.execute_reply.started":"2022-04-15T00:17:13.386285Z","shell.execute_reply":"2022-04-15T00:17:13.410125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Data Type for image: {type(img)}\")\nprint(f\"Shape of images dataset: {img.size()}\")\nprint(f\"Data Type for label: {type(label)}\")\nprint(f\"Label {sorted(train_df.loc[:, 'label'].unique())}\")","metadata":{"id":"Q1etS2PQh5Wd","outputId":"70e56539-65dc-4490-a1f4-674e57ee4860","execution":{"iopub.status.busy":"2022-04-15T00:17:13.412473Z","iopub.execute_input":"2022-04-15T00:17:13.413967Z","iopub.status.idle":"2022-04-15T00:17:13.427404Z","shell.execute_reply.started":"2022-04-15T00:17:13.413921Z","shell.execute_reply":"2022-04-15T00:17:13.426020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Digits Visualization</h2>","metadata":{"id":"WePKN7FMh5Wf"}},{"cell_type":"code","source":"img = next(iter(test_dataloader))","metadata":{"id":"AJlvuPjutYhr","execution":{"iopub.status.busy":"2022-04-15T00:17:13.429540Z","iopub.execute_input":"2022-04-15T00:17:13.430108Z","iopub.status.idle":"2022-04-15T00:17:13.487309Z","shell.execute_reply.started":"2022-04-15T00:17:13.430060Z","shell.execute_reply":"2022-04-15T00:17:13.486243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Visualizing of the one image in the test_dataset\nimg = next(iter(test_dataloader.dataset))\ncmap = plt.get_cmap('Blues')\nplt.imshow(img.squeeze(), cmap=cmap)\nplt.axis('off')\nplt.show()","metadata":{"id":"6QVZuDKCh5Wg","outputId":"6f811cd0-37df-4126-dbbc-3eb2b352f035","execution":{"iopub.status.busy":"2022-04-15T00:17:13.489663Z","iopub.execute_input":"2022-04-15T00:17:13.490732Z","iopub.status.idle":"2022-04-15T00:17:13.632185Z","shell.execute_reply.started":"2022-04-15T00:17:13.490686Z","shell.execute_reply":"2022-04-15T00:17:13.630998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Subplot instance\nfig, ax = plt.subplots(figsize=(8, 8))\n\n# Number of columns and rows\nNROW = 3\nNCOL = 3\n\n# loop over the range of NROW * NCOL starting from 1\nfor i in range(1, NROW * NROW + 1):\n    fig.add_subplot(NROW, NCOL, i)\n    \n    # Generat random number in range of length of image in the train_dataloader.dataset\n    idx = torch.randint(len(train_dataloader.dataset), size=(1,)).item()\n\n    # Get the image from the generated index\n    img, label = train_dataloader.dataset[idx]\n\n    # Plot it\n    plt.imshow(img.squeeze(), cmap=cmap)\n    plt.axis('off')\n    plt.title(label)\n    ax.set_xticklabels([])\n    ax.set_yticklabels([])\n    ax.set_axis_off()","metadata":{"id":"3rGh7K44h5Wh","outputId":"d4271f96-c5d4-4937-abb2-31c3e053cf65","execution":{"iopub.status.busy":"2022-04-15T00:17:13.637527Z","iopub.execute_input":"2022-04-15T00:17:13.637981Z","iopub.status.idle":"2022-04-15T00:17:14.439085Z","shell.execute_reply.started":"2022-04-15T00:17:13.637929Z","shell.execute_reply":"2022-04-15T00:17:14.438071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Model Building</h2>","metadata":{"id":"_yz-frsOh5Wi"}},{"cell_type":"code","source":"\nclass Model(nn.Module):\n    def __init__(self) -> None:\n        super(Model, self).__init__()\n\n        # Transpose Input data\n        self.flatten = nn.Flatten()\n\n        # Input layer\n        self.input_layer = nn.Linear(in_features=28*28, out_features=256)\n        self.first_relu = nn.ReLU() # Activation function\n        self.dropout_1 = nn.Dropout(.4) # Drop same pixal\n\n        # First hidden layer\n        self.first_hidden_layer = nn.Linear(in_features=256, out_features=256)\n        self.second_relu = nn.ReLU() # Activation function\n        self.dropout_2 = nn.Dropout(.4) # Drop same pixal\n\n        # Output layer\n        self.output_layer = nn.Linear(in_features=256, out_features=10)\n        self.softmax = nn.Softmax()\n    \n    def forward(self, X):\n        flatten = self.flatten(X)\n\n        # Input layer\n        input_layer = self.input_layer(flatten)\n        first_relu = self.first_relu(input_layer)\n        dropout_1 = self.dropout_1(first_relu)\n        \n        # First hidden layer\n        first_hidden_layer = self.first_hidden_layer(dropout_1)\n        second_relu = self.second_relu(first_hidden_layer)\n        dropout_2 = self.dropout_2(second_relu)\n\n        # Output layer\n        output_layer = self.output_layer(dropout_2)\n        logit = self.softmax(output_layer)\n        return logit","metadata":{"id":"k7peuEP0h5Wj","execution":{"iopub.status.busy":"2022-04-15T00:17:14.440905Z","iopub.execute_input":"2022-04-15T00:17:14.441448Z","iopub.status.idle":"2022-04-15T00:17:14.454438Z","shell.execute_reply.started":"2022-04-15T00:17:14.441399Z","shell.execute_reply":"2022-04-15T00:17:14.453288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Call the Model instance: model\nmnist_model = Model().to(device=device)\nsummary(mnist_model)","metadata":{"id":"O5Qyfsnwh5Wk","outputId":"db80fcec-db66-441a-ca13-90681d848e5a","execution":{"iopub.status.busy":"2022-04-15T00:17:14.456182Z","iopub.execute_input":"2022-04-15T00:17:14.456664Z","iopub.status.idle":"2022-04-15T00:17:18.033460Z","shell.execute_reply.started":"2022-04-15T00:17:14.456595Z","shell.execute_reply":"2022-04-15T00:17:18.032336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Compile The Model</h2>","metadata":{"id":"elytvylMh5Wl"}},{"cell_type":"code","source":"# Initialize the Adam Optimizer\noptimizer_fn = torch.optim.Adam(mnist_model.parameters())\n\n# Calling the \nloss_fn = nn.CrossEntropyLoss()","metadata":{"id":"M8gr746Kh5Wm","execution":{"iopub.status.busy":"2022-04-15T00:17:18.035427Z","iopub.execute_input":"2022-04-15T00:17:18.035751Z","iopub.status.idle":"2022-04-15T00:17:18.042314Z","shell.execute_reply.started":"2022-04-15T00:17:18.035709Z","shell.execute_reply":"2022-04-15T00:17:18.040864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2 style=\"font-size: 20px;\">Training The Fuction</h2>","metadata":{"id":"45qlSEl_h5Wm"}},{"cell_type":"code","source":"\nclass TrainEvaluateModel:\n    def __init__(\n           self,\n           model: any,\n           optimizer: any = None,\n           loss: any = nn.CrossEntropyLoss(),\n           lr: float = LEARNING_RATE,\n           device: any = None\n          ) -> None:\n        \"\"\"\n        Train and evaluate the model \n        :param model to train:\n        :optim for optimization process:\n        :param loss to adjust gradient from optimizer:\n        \"\"\"\n        self.model = model\n        self.lr = lr\n        self.optim = optimizer or torch.optim.Adam(\n                                                   mnist_model.parameters(), \n                                                   lr=self.lr)\n        self.loss = loss\n        self.device = device\n    \n    def train(self, data_loader) -> None:\n        \"\"\"\n        Train model on train_data\n        \"\"\"\n        # Indicating the model to training\n        self.model.train()\n\n        # Number of images in  data_loader: size \n        size = len(data_loader.dataset)\n\n        # Initialize the metric variable\n        total_acc, total_loss, count_label, current = 0, 0, 0, 0\n\n        start = time.time()\n        # iterate over the data_loader\n        for batch, (X, y) in enumerate(data_loader):\n            # Switch to device \n            X, y = X.to(self.device), y.to(self.device)\n       \n            # Make prediction\n            yhat = self.model(X)\n            \n            # *** Backpropagation Process ***\n\n            # Compute error by measure the degree of dissimilarity \n            # from obtained result in target\n            criterion = self.loss(yhat, y)\n            \n            # Reset the gradient of the model parameters\n            # Gradients by default add up; to prevent double-counting, \n            # we explicitly zero them at each iteration.\n            self.optim.zero_grad()\n      \n            # Back propagate the prediction loss to deposit the gradient of loss\n            # for learnable parameters\n            criterion.backward()\n\n\n            # Adjust the parameters by gradient collected in the backward pass\n            self.optim.step()\n\n            # Count number of labels\n            count_label += len(y)\n\n            # sum each loss to total_loss variable\n            total_loss += criterion.item()\n\n            _, predict = torch.max(yhat, 1)\n\n            # Add every accuracy on total_acc\n            total_acc += (predict == y).sum().item()\n            \n            if batch % 32 == 0:\n                print(\".\", end=\" \")\n                current += (batch / size)\n\n\n        stop = time.time()\n        time_taken = round(stop - start, 3)\n\n        return (total_loss / count_label,\n                 total_acc / count_label, time_taken,\n                int(round(current * 100))\n                )\n\n    def evaluate (self, data_loader) -> None:\n        \"\"\" \n        Evaluation model with validation data\n        \"\"\"\n        # Directing model to evaluation process\n        self.model.eval()\n\n        # Instaniate metric variables\n        total_loss, total_acc, count_labels = 0, 0, 0\n\n        # Disabling gradient calculation\n        with torch.no_grad():\n            for X, y in data_loader:\n                # Set to device\n                X, y = X.to(self.device), y.to(self.device)\n\n                # Make prediction\n                pred = self.model(X)\n\n                # Compute the loss(error)\n                criterion = self.loss(pred, y)\n\n                # Add number of label to count_labels\n                count_labels += len(y)\n\n                # Add criterion loss to total_loss\n                total_loss += criterion.item()\n\n                # Sum accuracy to total_acc\n                total_acc += (pred.argmax(1) == y).sum().item()\n            \n            # Finally return total_loss and total_acc which each is divded by\n            # count_labels\n            return total_loss / count_labels, total_acc /count_labels\n\n    def fit(\n        self, \n        train_data: DataLoader,\n        epochs: int = 1,\n        validation_data: DataLoader = None \n        ) -> None:\n        \"\"\"\n        The Fit method make use of train data and\n        validation data if provided\n        :param train_data. Data for training the model:\n        :param epoch. Number of iteration:\n        :param validation_data. Data to validate how well the model preformed:\n        \"\"\"\n        # Initializing variable for storing metric score\n        metrics = {}\n        acc_list = []\n        loss_list = []\n        valid_acc_list = []\n        valid_loss_list = []\n\n        # loop throught the epoch\n        for epoch in range(epochs):\n            print(f\"\\nEpoch {epoch + 1}/{epochs} \")\n            train = self.train(train_data)\n            # Instaniate train loss and accuracy\n            train_loss = round(train[0], 6)\n            train_acc = round(train[1], 5)\n            print(f\"{int(train[2])}s {train[3]}ms/steps - loss: {train_loss} - acc: {train_acc} \", end = \"\")\n            # Storing the model score\n            acc_list.append(train_acc)\n            loss_list.append(train_loss)\n\n            if validation_data:\n                valid = self.evaluate(validation_data)\n                # Instaniate train loss and accuracy\n                valid_loss = round(valid[0], 6)\n                valid_acc = round(valid[1], 4)\n                print(f\"- val_loss: {valid_loss} - val_acc: {valid_acc} \", end = \"\")\n                # Store the score \n                valid_loss_list.append(valid_loss)\n                valid_acc_list.append(valid_acc)\n\n        metrics[\"acc\"] = acc_list\n        metrics[\"loss\"] = loss_list\n        if validation_data:\n            metrics[\"val_acc\"] = valid_acc_list\n            metrics[\"val_loss\"] = valid_loss_list\n        return metrics\n\n    def __repr__(self):\n        return \"{}\".format({\n            \"model\": self.model,\n            \"learning_rate\": self.lr,\n            \"optimizer\": self.optim, \n            \"loss\": self.loss,\n            \"device\": self.device\n            })\n    \n    def predict(self, y) -> torch.tensor:\n        # list storage for predictions\n        predictions = []\n\n        # Indicate to evalation process\n        #self.model.eval()\n\n        # Don't use the gradient\n        with torch.no_grad():\n\n            # Loop over the values in y\n            for val in y.dataset:\n\n                # switch to device\n                val = val.to(self.device)\n\n                # Make prediction\n                probability = self.model(val)\n\n                # probability variable returns probability\n                # Therefor convert it to actual value\n                pred = torch.argmax(probability, 1).item()\n\n                # Add prediction to predictions list\n                predictions.append(pred)\n        return predictions\n\n                \n\n\n            \n\n\n\n\n\n\n","metadata":{"id":"-VQ-HlOR8xGi","execution":{"iopub.status.busy":"2022-04-15T00:17:18.044344Z","iopub.execute_input":"2022-04-15T00:17:18.045145Z","iopub.status.idle":"2022-04-15T00:17:18.079197Z","shell.execute_reply.started":"2022-04-15T00:17:18.045098Z","shell.execute_reply":"2022-04-15T00:17:18.078071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize the TrainEvaluationModel: train_eval\ntrain_eval = TrainEvaluateModel(\n                   model=mnist_model,\n                   device=device\n               )\ntrain_eval","metadata":{"id":"mqoD-ZsXQyXS","outputId":"d81907d6-a134-4a88-983a-3f0c26919cac","execution":{"iopub.status.busy":"2022-04-15T00:17:18.081275Z","iopub.execute_input":"2022-04-15T00:17:18.081964Z","iopub.status.idle":"2022-04-15T00:17:18.098217Z","shell.execute_reply.started":"2022-04-15T00:17:18.081918Z","shell.execute_reply":"2022-04-15T00:17:18.096994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = train_eval.fit(train_dataloader, epochs=20, validation_data=valid_dataloader)","metadata":{"id":"kjX0dNVt1rr4","outputId":"37bbd1a6-584a-4927-e89c-cb6c099dec2e","execution":{"iopub.status.busy":"2022-04-15T00:17:18.100237Z","iopub.execute_input":"2022-04-15T00:17:18.100786Z","iopub.status.idle":"2022-04-15T00:26:08.150478Z","shell.execute_reply.started":"2022-04-15T00:17:18.100737Z","shell.execute_reply":"2022-04-15T00:26:08.148883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_metrics(history: Dict, n_epoch: int) -> plt.figure:\n    \"\"\"\n    Return matplotlib figure with accuracy score \n    and loss score visualization\n    parameter:\n    -----------------------\n    :param history, Dictionary with model loss and accuracy:\n    :return plt.figure with subplots of train and validation score:\n    \"\"\"\n    fig, ax = plt.subplots(1, 2, figsize=(12, 5))\n    epochs = list(range(n_epoch))\n    ax[0].plot(epochs, history[\"acc\"], \"ob\")\n    ax[0].plot(epochs, history[\"val_acc\"])\n\n    ax[1].plot(epochs, history[\"loss\"], \"ob\")\n    ax[1].plot(epochs, history[\"val_loss\"])\n\n    for i in range(2):\n        ax[i].set_ylabel(f\"Accuracy\")\n        ax[i].set_xlabel(\"Epoch\")\n        \n        # Hide the right and top spines\n        ax[i].spines['right'].set_visible(False)\n        ax[i].spines['top'].set_visible(False)\n        ax[i].legend([\"train\", \"valid\"])\n\n    ax[0].set_title(f\"Train And Valid Accuracy\".title(), weight=500, y=1.06, size=15)\n    ax[1].set_title(f\"Train And Valid loss\".title(), weight=500, y=1.06, size=15)\n    fig.suptitle(\"Model Score Visualization\", y=1.1, weight=800, size=17)\nplot_metrics(history, 20)","metadata":{"id":"rBP5eUgpq2AF","outputId":"9185fc45-6774-475b-86c2-b1fe843419b1","execution":{"iopub.status.busy":"2022-04-15T00:26:08.152147Z","iopub.execute_input":"2022-04-15T00:26:08.152703Z","iopub.status.idle":"2022-04-15T00:26:08.706863Z","shell.execute_reply.started":"2022-04-15T00:26:08.152655Z","shell.execute_reply":"2022-04-15T00:26:08.705758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"id":"KQ-pYvS5sXnL"}},{"cell_type":"code","source":"# make predictions\npred = train_eval.predict(test_dataloader)","metadata":{"id":"Nj28nVG1hx2A","execution":{"iopub.status.busy":"2022-04-15T00:26:08.708852Z","iopub.execute_input":"2022-04-15T00:26:08.709473Z","iopub.status.idle":"2022-04-15T00:26:35.189477Z","shell.execute_reply.started":"2022-04-15T00:26:08.709425Z","shell.execute_reply":"2022-04-15T00:26:35.188457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv(folder_path + '/sample_submission.csv')\nsubmission['Label']=pred\nsubmission.to_csv('pytorch_prediction6.csv', index=False)","metadata":{"id":"EHpXI36ak1Ia","execution":{"iopub.status.busy":"2022-04-15T00:26:35.191308Z","iopub.execute_input":"2022-04-15T00:26:35.191623Z","iopub.status.idle":"2022-04-15T00:26:35.273308Z","shell.execute_reply.started":"2022-04-15T00:26:35.191579Z","shell.execute_reply":"2022-04-15T00:26:35.272306Z"},"trusted":true},"execution_count":null,"outputs":[]}]}