{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## SET UP","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom pathlib import Path\n\nfrom tqdm import tqdm\n\nimport gc\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom torch.utils.data import TensorDataset, Dataset, DataLoader\nimport torchvision.transforms.v2 as transforms\nfrom torchvision.models import get_model","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:25:38.569058Z","iopub.execute_input":"2023-07-17T14:25:38.569684Z","iopub.status.idle":"2023-07-17T14:25:44.510949Z","shell.execute_reply.started":"2023-07-17T14:25:38.569645Z","shell.execute_reply":"2023-07-17T14:25:44.509920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparation the data","metadata":{}},{"cell_type":"code","source":"main_path = Path('../input/digit-recognizer')\ntrain_csv = pd.read_csv(main_path/'train.csv')\n\nprint(\"Shape train.csv:\", train_csv.shape)\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:51.937426Z","iopub.execute_input":"2023-07-17T14:27:51.938324Z","iopub.status.idle":"2023-07-17T14:27:55.627470Z","shell.execute_reply.started":"2023-07-17T14:27:51.938284Z","shell.execute_reply":"2023-07-17T14:27:55.626471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see how many targets there are","metadata":{}},{"cell_type":"code","source":"labels = sorted(train_csv.label.unique())\n\nprint(\"Number of labels:\", len(labels))\nlabels","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:55.629829Z","iopub.execute_input":"2023-07-17T14:27:55.630590Z","iopub.status.idle":"2023-07-17T14:27:55.643775Z","shell.execute_reply.started":"2023-07-17T14:27:55.630553Z","shell.execute_reply":"2023-07-17T14:27:55.642579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see there 10 labels (10 digits from 0 to 9)","metadata":{}},{"cell_type":"markdown","source":"Let's take a look at the test.csv","metadata":{}},{"cell_type":"code","source":"test_csv = pd.read_csv(main_path/'test.csv')\n\nprint(\"Shape test.csv:\", test_csv.shape)\ntest_csv.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:55.645312Z","iopub.execute_input":"2023-07-17T14:27:55.645972Z","iopub.status.idle":"2023-07-17T14:27:57.726277Z","shell.execute_reply.started":"2023-07-17T14:27:55.645934Z","shell.execute_reply":"2023-07-17T14:27:57.725208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look some figures","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 3)\nax = axes.ravel()\n\nfor i in range(6):\n    ax[i].imshow(train_csv.iloc[i, 1:].values.reshape(28,28), cmap='gray')\n    ax[i].set_title(train_csv.label[i])\n    ax[i].axis('off')\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:57.729315Z","iopub.execute_input":"2023-07-17T14:27:57.729799Z","iopub.status.idle":"2023-07-17T14:27:58.175968Z","shell.execute_reply.started":"2023-07-17T14:27:57.729760Z","shell.execute_reply":"2023-07-17T14:27:58.174856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DataLoaders","metadata":{}},{"cell_type":"markdown","source":"First of all, let's create a function to divide the train.csv into train and validation","metadata":{}},{"cell_type":"code","source":"X = train_csv.iloc[:,1:].values.reshape(-1,1,28,28)\ny = train_csv.label.values\n\nX = torch.Tensor(X)\ny = torch.Tensor(y).type(torch.LongTensor)\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:58.177408Z","iopub.execute_input":"2023-07-17T14:27:58.177896Z","iopub.status.idle":"2023-07-17T14:27:58.953012Z","shell.execute_reply.started":"2023-07-17T14:27:58.177844Z","shell.execute_reply":"2023-07-17T14:27:58.951936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's create the DataLoaders. To do that, the easiest way is to create a class that inherites from Dataset:","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    \n    def __init__(self, X, y, transform=None):\n        self.X = X\n        self.y = y\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, index):\n        x = self.X[index]\n        y = self.y[index]\n        \n        if self.transform:\n            x = self.transform(x)\n            \n        return x,y","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:58.954662Z","iopub.execute_input":"2023-07-17T14:27:58.955094Z","iopub.status.idle":"2023-07-17T14:27:58.963053Z","shell.execute_reply.started":"2023-07-17T14:27:58.955061Z","shell.execute_reply":"2023-07-17T14:27:58.961679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformation for training\ntrans_train = transforms.Compose([\n                    transforms.ToPILImage(),\n                    transforms.Resize(94),\n                    transforms.Grayscale(num_output_channels=3),\n                    transforms.RandomRotation(15),\n                    transforms.RandomAffine(0, translate=(0.1,0.1)),\n                    transforms.ToTensor(),\n                    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n    \n#Transformation for validation and test\ntrans_valid = transforms.Compose([\n                    transforms.ToPILImage(),\n                    transforms.Resize(94),\n                    transforms.Grayscale(num_output_channels=3),\n                    transforms.ToTensor(),\n                    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# Dataset after transformation\ndataset_train = CustomDataset(X_train, y_train, transform=trans_train)\ndataset_valid = CustomDataset(X_valid, y_valid, transform=trans_valid)\n\n# Dataloaders\nbs = 32\nloaders = {\"train\": DataLoader(dataset_train, batch_size = bs),\n           \"valid\": DataLoader(dataset_valid, batch_size = bs)}","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:58.964877Z","iopub.execute_input":"2023-07-17T14:27:58.965603Z","iopub.status.idle":"2023-07-17T14:27:58.977055Z","shell.execute_reply.started":"2023-07-17T14:27:58.965573Z","shell.execute_reply":"2023-07-17T14:27:58.976124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"markdown","source":"- Resnet34","metadata":{}},{"cell_type":"code","source":"resnet34 = get_model('resnet34', weights=\"DEFAULT\")\n\nfor param in resnet34.parameters():\n    param.requires_grad = False\n\n# Get the number of features of the last layer\nnum_features = resnet34.fc.in_features\n\n# Modify the last layer\nresnet34.fc = nn.Linear(num_features, len(labels))","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:27:58.978248Z","iopub.execute_input":"2023-07-17T14:27:58.978996Z","iopub.status.idle":"2023-07-17T14:28:00.417005Z","shell.execute_reply.started":"2023-07-17T14:27:58.978927Z","shell.execute_reply":"2023-07-17T14:28:00.416029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Convnext_tiny","metadata":{}},{"cell_type":"code","source":"convnext_tiny = get_model('convnext_tiny', weights=\"DEFAULT\")\n\nfor param in convnext_tiny.parameters():\n    param.requires_grad = False\n    \n# Get the number of features of the last layer\nnum_features = convnext_tiny.classifier[-1].in_features\n\n# Modify the last layer\nconvnext_tiny.classifier[-1] = nn.Linear(num_features, len(labels))","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:28:00.418594Z","iopub.execute_input":"2023-07-17T14:28:00.419006Z","iopub.status.idle":"2023-07-17T14:28:02.815963Z","shell.execute_reply.started":"2023-07-17T14:28:00.418971Z","shell.execute_reply":"2023-07-17T14:28:02.814775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Efficientnet_b3","metadata":{}},{"cell_type":"code","source":"efficientnet_b3 = get_model('efficientnet_b3', weights=\"DEFAULT\")\n\nfor param in efficientnet_b3.parameters():\n    param.requires_grad = False\n    \n# Get the number of features of the last layer\nnum_features = efficientnet_b3.classifier[-1].in_features\n\n# Modify the last layer\nefficientnet_b3.classifier[-1] = nn.Linear(num_features, len(labels))","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:28:02.819070Z","iopub.execute_input":"2023-07-17T14:28:02.820003Z","iopub.status.idle":"2023-07-17T14:28:03.906385Z","shell.execute_reply.started":"2023-07-17T14:28:02.819974Z","shell.execute_reply":"2023-07-17T14:28:03.905401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [resnet34, convnext_tiny, efficientnet_b3]","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:58:11.483219Z","iopub.execute_input":"2023-07-17T14:58:11.483570Z","iopub.status.idle":"2023-07-17T14:58:11.488851Z","shell.execute_reply.started":"2023-07-17T14:58:11.483542Z","shell.execute_reply":"2023-07-17T14:58:11.487942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this training function, we first use the network as **fixed feature extractor**, and then **finetune** it.\nIn addition, we use Early Stopping because sometimes the network may not be learning and it's not useful to keep training it","metadata":{}},{"cell_type":"code","source":"def train(model, epochs, patience, loaders):\n    \n    print(f\"Model: {model.__class__.__name__}\")\n    \n    criterion = nn.CrossEntropyLoss()\n    \n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model = model.to(device)\n    \n    best_val_loss = float('inf')\n    my_patience = 0\n    best_model_weights = None\n    list_train_loss = []\n    list_valid_loss = []\n    list_train_acc = []\n    list_valid_acc = []\n    \n    for tf in ['extractor', 'finetuning']:\n        \n        # Fixed feature extractor \n        if tf == 'extractor':\n            print(\"NETWORK as FIXED FEATURE EXTRACTOR\")\n            \n            if model.__class__.__name__=='ResNet':\n                optimizer = torch.optim.Adam(model.fc.parameters(), lr=0.001)\n            else:\n                optimizer = torch.optim.Adam(model.classifier.parameters(), lr=0.001)\n                \n        # Finetuning\n        else:\n            print(\"FINETUNING THE NETWORK\")\n            for param in model.parameters():\n                param.requires_grad = True\n            optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n            \n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n                               \n        for epoch in range(epochs):\n            print(f\"Epoch {epoch+1}/{epochs}\")\n\n            for phase in ['train', 'valid']:\n                if phase=='train':\n                    model.train()    # Set model to training mode\n                else:\n                    model.eval()     # Set model to evaluate mode\n\n                running_loss = 0.0\n                running_corrects = 0\n\n                # Training\n                train_loss = 0\n                for inputs,labels in tqdm(loaders[phase]):\n                    inputs = inputs.to(device)\n                    labels = labels.to(device)\n\n                    optimizer.zero_grad()\n\n                    # In train phase we enable gradients and in eval we disable it to save resources\n                    with torch.set_grad_enabled(phase=='train'):\n                        outputs = model(inputs)\n                        loss = criterion(outputs, labels)\n                        preds = torch.argmax(outputs, dim=1)\n\n                        # Backward + optimize if in training phase\n                        if phase == 'train':\n                            loss.backward()\n                            optimizer.step()\n\n                    # Stadistics\n                    running_loss += loss.item() * inputs.size(0)\n                    running_corrects += torch.sum(preds == labels.data)\n                    \n                epoch_loss = running_loss / len(loaders[phase].dataset)\n                epoch_acc = running_corrects.double() / len(loaders[phase].dataset)\n                \n                if phase == 'train':\n                    scheduler.step()\n                    list_train_loss.append(epoch_loss)\n                    list_train_acc.append(epoch_acc.item())\n                else:\n                    list_valid_loss.append(epoch_loss)\n                    list_valid_acc.append(epoch_acc.item())\n\n                print(f\"{phase} Loss: {epoch_loss:.4f} | {phase} Accuracy: {epoch_acc:.4f}\")\n\n            # Early Stopping\n            if list_valid_loss[-1] < best_val_loss:\n                my_patience = 0\n                best_val_loss = list_valid_loss[-1]\n                best_model_weights = model.state_dict()\n            else:\n                my_patience += 1\n                if my_patience >= patience:\n                    print(f\"*** Patience={patience} reached ***\")\n                    break\n\n            print(\"-\"*50)\n                \n        model.load_state_dict(best_model_weights)\n    \n    # Empty cache from GPU\n    gc.collect()\n    torch.cuda.empty_cache()\n    \n    print(\"=\"*70 + \"\\n\")\n        \n    return model, list_train_loss, list_valid_loss, list_train_acc, list_valid_acc","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:58:12.744971Z","iopub.execute_input":"2023-07-17T14:58:12.745327Z","iopub.status.idle":"2023-07-17T14:58:12.767048Z","shell.execute_reply.started":"2023-07-17T14:58:12.745299Z","shell.execute_reply":"2023-07-17T14:58:12.765832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lists to store the loss and the accuracy in order to make graphs later\ntrain_losses = []\nvalid_losses = []\ntrain_accs = []\nvalid_accs = []\n\nmodels_trained = []\n\nfor model in models:\n    model_trained, train_losses_model, valid_losses_model, train_accs_model, valid_accs_model = train(model, 5, 2, loaders)\n    \n    models_trained.append(model_trained)\n    train_losses.append(train_losses_model)\n    valid_losses.append(valid_losses_model)\n    train_accs.append(train_accs_model)\n    valid_accs.append(valid_accs_model)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T14:58:19.064390Z","iopub.execute_input":"2023-07-17T14:58:19.065285Z","iopub.status.idle":"2023-07-17T15:53:43.968204Z","shell.execute_reply.started":"2023-07-17T14:58:19.065241Z","shell.execute_reply":"2023-07-17T15:53:43.967120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Graphs","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(3,2, figsize=(10,7))\n\ni=0\nfor model, train_loss, valid_loss, train_acc, valid_acc in zip(models_trained, train_losses, valid_losses, train_accs, valid_accs):\n    j=0\n    \n    epochs_train = range(1, len(train_loss)+1)\n    epochs_valid = range(1, len(valid_loss)+1)\n    \n    ax[i, j].plot(epochs_train, train_loss, label='Train loss')\n    ax[i, j].plot(epochs_valid, valid_loss, label='Valid loss')\n    \n    ax[i,j].set_title(model.__class__.__name__, fontweight=\"bold\")\n    ax[i,j].set_xlabel('Epochs')\n    ax[i,j].set_ylabel('Loss')\n    ax[i,j].legend(loc='best')\n    ax[i,j].grid()\n    \n    j+=1\n    \n    epochs_train = range(1, len(train_acc)+1)\n    epochs_valid = range(1, len(valid_acc)+1)\n    \n    ax[i, j].plot(epochs_train, train_acc, label='Train Accuracy')\n    ax[i, j].plot(epochs_valid, valid_acc, label='Valid Accuracy')\n    \n    ax[i,j].set_title(model.__class__.__name__, fontweight=\"bold\")\n    ax[i,j].set_xlabel('Epochs')\n    ax[i,j].set_ylabel('Acc')\n    ax[i,j].legend(loc='best')\n    ax[i,j].grid()\n    \n    i+=1\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:53:48.542698Z","iopub.execute_input":"2023-07-17T15:53:48.543071Z","iopub.status.idle":"2023-07-17T15:53:49.952006Z","shell.execute_reply.started":"2023-07-17T15:53:48.543041Z","shell.execute_reply":"2023-07-17T15:53:49.950824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"markdown","source":"### Predictions **(ENSEMBLE)**","metadata":{}},{"cell_type":"markdown","source":"Let's create de class for the Test dataset. It's the same than the previous class `CustomDataset` but without target","metadata":{}},{"cell_type":"code","source":"class CustomDatasetTest(Dataset):\n    \n    def __init__(self, X, transform=None):\n        self.X = X\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, index):\n        x = self.X[index]\n        \n        if self.transform:\n            x = self.transform(x)\n            \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:54:41.355892Z","iopub.execute_input":"2023-07-17T15:54:41.356260Z","iopub.status.idle":"2023-07-17T15:54:41.362673Z","shell.execute_reply.started":"2023-07-17T15:54:41.356232Z","shell.execute_reply":"2023-07-17T15:54:41.361616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_csv.values.reshape(-1,1,28,28)\nX_test = torch.Tensor(X_test)\n\ndataset_test = CustomDatasetTest(X_test, transform=trans_valid)\n\n# Dataloader\nbs = 100\ntest_loader = DataLoader(dataset_test, batch_size = bs)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:54:41.854306Z","iopub.execute_input":"2023-07-17T15:54:41.854674Z","iopub.status.idle":"2023-07-17T15:54:41.938152Z","shell.execute_reply.started":"2023-07-17T15:54:41.854643Z","shell.execute_reply":"2023-07-17T15:54:41.936653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's make the predictions","metadata":{}},{"cell_type":"code","source":"predictions = []\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nfor model in models:\n    \n    prediction_model = []\n    for X in test_loader:\n            \n        X = X.to(device)\n    \n        # Probabilities of each digit for every test figure\n        output = model(X)\n\n        # Take the digit with the highest probability\n        prediction = torch.argmax(output, dim=1)\n        \n        prediction_model += prediction\n    \n    predictions.append(prediction_model)\n    \npredictions = torch.Tensor(predictions)\npredictions.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:54:42.448184Z","iopub.execute_input":"2023-07-17T15:54:42.449089Z","iopub.status.idle":"2023-07-17T15:55:53.466591Z","shell.execute_reply.started":"2023-07-17T15:54:42.449044Z","shell.execute_reply":"2023-07-17T15:55:53.465646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Each row is the result of each model, and each column a different test, therefore, to obtain the final prediction we make the mode of each column.","metadata":{}},{"cell_type":"code","source":"final_prediction = torch.mode(predictions, dim=0).values.to(torch.int32)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:55:53.468540Z","iopub.execute_input":"2023-07-17T15:55:53.469157Z","iopub.status.idle":"2023-07-17T15:55:53.476356Z","shell.execute_reply.started":"2023-07-17T15:55:53.469122Z","shell.execute_reply":"2023-07-17T15:55:53.475434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submit","metadata":{}},{"cell_type":"code","source":"ss = pd.read_csv(main_path/'sample_submission.csv')\nss","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:55:59.716740Z","iopub.execute_input":"2023-07-17T15:55:59.717118Z","iopub.status.idle":"2023-07-17T15:55:59.742929Z","shell.execute_reply.started":"2023-07-17T15:55:59.717088Z","shell.execute_reply":"2023-07-17T15:55:59.741925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_working = Path('/kaggle/working')\nss['Label'] = final_prediction\nss.to_csv(path_working/'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:56:02.959949Z","iopub.execute_input":"2023-07-17T15:56:02.960419Z","iopub.status.idle":"2023-07-17T15:56:03.060089Z","shell.execute_reply.started":"2023-07-17T15:56:02.960379Z","shell.execute_reply":"2023-07-17T15:56:03.058892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-07-17T15:56:03.845246Z","iopub.execute_input":"2023-07-17T15:56:03.845613Z","iopub.status.idle":"2023-07-17T15:56:04.883670Z","shell.execute_reply.started":"2023-07-17T15:56:03.845573Z","shell.execute_reply":"2023-07-17T15:56:04.882408Z"},"trusted":true},"execution_count":null,"outputs":[]}]}