{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Purpose of this Notebook","metadata":{}},{"cell_type":"markdown","source":"This notebook is just a basic one to show how one might train a classifier on the mnist dataset with Pytorch. The chosen classifier is a CNN, as we try to classify images with simple patterns (no need to go to VIT architecture). \\\n\nThe goal is to showcase the basic components of model training (from scratch) in PyTorch : no regularization, no data augmentation, no Transfer learning, no learning rate scheduling, no hyper-parameters tuning. \\\n\nI refer you to later notebooks for fancier techniques.","metadata":{}},{"cell_type":"markdown","source":"## Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torchsummary import summary\n\nfrom sklearn.model_selection import train_test_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:03.815685Z","iopub.execute_input":"2025-01-28T19:22:03.815974Z","iopub.status.idle":"2025-01-28T19:22:03.820243Z","shell.execute_reply.started":"2025-01-28T19:22:03.815951Z","shell.execute_reply":"2025-01-28T19:22:03.819283Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Dataset","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nprint(df_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:03.855481Z","iopub.execute_input":"2025-01-28T19:22:03.855721Z","iopub.status.idle":"2025-01-28T19:22:05.578801Z","shell.execute_reply.started":"2025-01-28T19:22:03.855701Z","shell.execute_reply":"2025-01-28T19:22:05.578042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = df_train['label'].values\nX = df_train.drop(labels='label', axis=1).values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:05.579757Z","iopub.execute_input":"2025-01-28T19:22:05.579971Z","iopub.status.idle":"2025-01-28T19:22:05.661754Z","shell.execute_reply.started":"2025-01-28T19:22:05.579951Z","shell.execute_reply":"2025-01-28T19:22:05.661045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.15)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:05.663544Z","iopub.execute_input":"2025-01-28T19:22:05.663849Z","iopub.status.idle":"2025-01-28T19:22:06.029909Z","shell.execute_reply.started":"2025-01-28T19:22:05.663821Z","shell.execute_reply":"2025-01-28T19:22:06.029163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model definition","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 256\n\ntorch_X_train = torch.from_numpy(X_train)\ntorch_y_train = torch.from_numpy(y_train)\n\n# create feature and targets tensor for test set.\ntorch_X_valid = torch.from_numpy(X_valid)\ntorch_y_valid = torch.from_numpy(y_valid)\n\ntorch_X_train = torch_X_train.view(-1, 1,28,28).float()\ntorch_X_valid = torch_X_valid.view(-1,1,28,28).float()\n\n# Pytorch train and test sets\ntrain = torch.utils.data.TensorDataset(torch_X_train,torch_y_train)\nvalid = torch.utils.data.TensorDataset(torch_X_valid,torch_y_valid)\n\n# data loader\ntrain_loader = torch.utils.data.DataLoader(train, batch_size = BATCH_SIZE, shuffle = False)\nvalid_loader = torch.utils.data.DataLoader(valid, batch_size = BATCH_SIZE, shuffle = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:06.031354Z","iopub.execute_input":"2025-01-28T19:22:06.031661Z","iopub.status.idle":"2025-01-28T19:22:06.114349Z","shell.execute_reply.started":"2025-01-28T19:22:06.031631Z","shell.execute_reply":"2025-01-28T19:22:06.113466Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Definition","metadata":{}},{"cell_type":"code","source":"class basic_CNN(nn.Module):\n    def __init__(self):\n        super(basic_CNN, self).__init__()\n        self.conv1 = nn.Conv2d(1,32,kernel_size=3,padding='same')\n        self.conv2 = nn.Conv2d(32,32,kernel_size=3,padding='same')\n        self.conv3 = nn.Conv2d(32,64,kernel_size=3,padding='same')\n        self.conv4 = nn.Conv2d(64,64,kernel_size=3,padding='same')\n        self.conv5 = nn.Conv2d(64,128,kernel_size=3,padding='same')\n        self.conv6 = nn.Conv2d(128,128,kernel_size=3,padding='same')\n\n        self.linear1 = nn.Linear(7*7*128, 64)\n        self.linear2 = nn.Linear(64, 10)\n\n    def forward(self, X):\n        X = F.leaky_relu(self.conv1(X))\n        X = F.leaky_relu(self.conv2(X))\n        X = nn.AvgPool2d(kernel_size=2)(X)\n        X = F.leaky_relu(self.conv3(X))\n        X = F.leaky_relu(self.conv4(X))\n        X = nn.AvgPool2d(kernel_size=2)(X)\n        X = F.leaky_relu(self.conv5(X))\n        X = F.leaky_relu(self.conv6(X))\n        X = nn.Flatten()(X)\n\n        X = F.leaky_relu(self.linear1(X))\n        X = self.linear2(X)\n\n        return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:06.115298Z","iopub.execute_input":"2025-01-28T19:22:06.115557Z","iopub.status.idle":"2025-01-28T19:22:06.122056Z","shell.execute_reply.started":"2025-01-28T19:22:06.115527Z","shell.execute_reply":"2025-01-28T19:22:06.121291Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training loop","metadata":{}},{"cell_type":"code","source":"def fit(model, train_data_loader, valid_data_loader, optimizer, epochs, loss_fn, device):\n    history = {}\n    train_size = len(train_data_loader.dataset)\n    valid_size = len(valid_data_loader.dataset)\n    for epoch in range(epochs):\n        print(\"epoch\", epoch+1,\"/\", epochs)\n        model.train()\n        train_loss_mean = 0\n        correct = 0\n        for batch_idx, (X, y_true) in enumerate(train_loader):\n            X = X.to(device)\n            y_true = y_true.to(device)\n            \n            y_pred = model(X)\n            loss = loss_fn(y_pred, y_true.long())\n\n            predicted = torch.max(y_pred.data, 1)[1] \n            correct += (predicted == y_true).sum()\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            train_loss_mean += loss.item()\n\n        train_loss_mean /= train_size\n        train_accuracy = (correct/train_size).item()\n\n        print(\"train loss mean :\", train_loss_mean)\n        \n        if \"loss\" in history.keys():\n            history[\"loss\"].append(train_loss_mean)\n        else:\n            history[\"loss\"] = [train_loss_mean]\n\n        if \"accuracy\" in history.keys():\n            history[\"accuracy\"].append(train_accuracy)\n        else:\n            history[\"accuracy\"] = [train_accuracy]\n\n        model.eval()\n\n        valid_loss_mean = 0\n        correct = 0\n        for batch_idx, (X, y_true) in enumerate(valid_loader):\n            X = X.to(device)\n            y_true = y_true.to(device)\n            \n            y_pred = model(X)\n            loss = loss_fn(y_pred, y_true.long())\n\n            predicted = torch.max(y_pred.data, 1)[1] \n            correct += (predicted == y_true).sum()\n\n            valid_loss_mean += loss.item()\n\n        valid_loss_mean /= valid_size\n        valid_accuracy = (correct/valid_size).item()\n\n        print(\"valid loss mean :\", valid_loss_mean)\n\n        if \"val_loss\" in history.keys():\n            history[\"val_loss\"].append(valid_loss_mean)\n        else:\n            history[\"val_loss\"] = [valid_loss_mean]\n\n        if \"val_accuracy\" in history.keys():\n            history[\"val_accuracy\"].append(valid_accuracy)\n        else:\n            history[\"val_accuracy\"] = [valid_accuracy]\n\n    return history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:06.122872Z","iopub.execute_input":"2025-01-28T19:22:06.123067Z","iopub.status.idle":"2025-01-28T19:22:06.140345Z","shell.execute_reply.started":"2025-01-28T19:22:06.123049Z","shell.execute_reply":"2025-01-28T19:22:06.139598Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model compilation","metadata":{}},{"cell_type":"code","source":"model = basic_CNN()\n\noptimizer = torch.optim.Adam(model.parameters())\nloss_fn = nn.CrossEntropyLoss(reduction='sum')\n\nepochs = 20","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:06.141124Z","iopub.execute_input":"2025-01-28T19:22:06.141367Z","iopub.status.idle":"2025-01-28T19:22:06.162322Z","shell.execute_reply.started":"2025-01-28T19:22:06.141347Z","shell.execute_reply":"2025-01-28T19:22:06.161469Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Compilation","metadata":{}},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using {device} device\")\n\nmodel.to(device)\n\nsummary(model, (1, 28, 28))\n\nhistory = fit(model, train_loader, valid_loader, optimizer, epochs, loss_fn, device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:06.163111Z","iopub.execute_input":"2025-01-28T19:22:06.163412Z","iopub.status.idle":"2025-01-28T19:22:48.879198Z","shell.execute_reply.started":"2025-01-28T19:22:06.163384Z","shell.execute_reply":"2025-01-28T19:22:48.878480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(np.log(history['loss']))\nplt.plot(np.log(history['val_loss']))\nplt.title('model log-loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:48.880920Z","iopub.execute_input":"2025-01-28T19:22:48.881147Z","iopub.status.idle":"2025-01-28T19:22:49.054929Z","shell.execute_reply.started":"2025-01-28T19:22:48.881117Z","shell.execute_reply":"2025-01-28T19:22:49.054151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(history['loss'])\nplt.plot(history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:49.055842Z","iopub.execute_input":"2025-01-28T19:22:49.056053Z","iopub.status.idle":"2025-01-28T19:22:49.231736Z","shell.execute_reply.started":"2025-01-28T19:22:49.056036Z","shell.execute_reply":"2025-01-28T19:22:49.230867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(history['accuracy'])\nplt.plot(history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-28T19:22:49.232652Z","iopub.execute_input":"2025-01-28T19:22:49.232961Z","iopub.status.idle":"2025-01-28T19:22:49.406273Z","shell.execute_reply.started":"2025-01-28T19:22:49.232929Z","shell.execute_reply":"2025-01-28T19:22:49.405420Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Results","metadata":{}},{"cell_type":"markdown","source":"As expected, the task is very easy and we get very high validation accuracy very fast. \\\nHowever we see that the model overfits : the training loss continues decreasing, whereas the validation loss stays constant. This indicates that the model learns features that are only useful for the training dataset. \\\nTraining on more data, or other techniques could reduce this overfitting. We will see some techniques in later notebooks.","metadata":{}}]}