{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-22T15:42:15.494753Z","iopub.execute_input":"2024-08-22T15:42:15.495227Z","iopub.status.idle":"2024-08-22T15:42:15.503623Z","shell.execute_reply.started":"2024-08-22T15:42:15.495187Z","shell.execute_reply":"2024-08-22T15:42:15.502659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Imports\nimport torch \nimport torch.nn as nn\nfrom torch.nn import functional as F \nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport torchvision\nfrom tqdm import tqdm, trange\nimport time\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:15.505215Z","iopub.execute_input":"2024-08-22T15:42:15.505492Z","iopub.status.idle":"2024-08-22T15:42:15.514012Z","shell.execute_reply.started":"2024-08-22T15:42:15.505468Z","shell.execute_reply":"2024-08-22T15:42:15.513250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Hyperparamters\nseed = 1320\nval_size = 0.2\nbatch_size = 128\ntorch.manual_seed(seed)\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:15.515119Z","iopub.execute_input":"2024-08-22T15:42:15.515859Z","iopub.status.idle":"2024-08-22T15:42:15.524042Z","shell.execute_reply.started":"2024-08-22T15:42:15.515834Z","shell.execute_reply":"2024-08-22T15:42:15.523187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Let's split the dataset to training and validation splits**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntrain, val = train_test_split(train, test_size = val_size)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:15.526431Z","iopub.execute_input":"2024-08-22T15:42:15.526761Z","iopub.status.idle":"2024-08-22T15:42:18.274418Z","shell.execute_reply.started":"2024-08-22T15:42:15.526731Z","shell.execute_reply":"2024-08-22T15:42:18.273628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Code preprocessing functions**","metadata":{}},{"cell_type":"code","source":"def normalize(df):\n    return df / 255\n\ndef reshape_im(df):\n    return df.values.reshape(-1, 28, 28, 1)\n\ndef to_tensor(df):\n    df = np.float32(df)\n    tensor = torch.tensor(df)\n    return tensor\n\ndef transform(df):\n    df = normalize(df)\n    df = reshape_im(df)\n    return to_tensor(df)\n\ndef transform_labels(label):\n    return torch.tensor(label, dtype=torch.long)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.275554Z","iopub.execute_input":"2024-08-22T15:42:18.275843Z","iopub.status.idle":"2024-08-22T15:42:18.282498Z","shell.execute_reply.started":"2024-08-22T15:42:18.275818Z","shell.execute_reply":"2024-08-22T15:42:18.281420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Create a custom pytorch dataset**","metadata":{}},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, data, test = False):\n        self.test = test\n        if not test:\n            self.image = transform(data.drop(\"label\", axis=1))\n            self.label = transform_labels(data[\"label\"].to_numpy())\n        else:\n            self.image = transform(data)\n            self.label = None\n    def __len__(self):\n        return len(self.image)\n    def __getitem__(self, idx):\n        if not self.test:\n            im, label = self.image[idx], self.label[idx]\n        else:\n            im, label = self.image[idx], torch.tensor(0)\n        return im, label\n        ","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.283555Z","iopub.execute_input":"2024-08-22T15:42:18.283791Z","iopub.status.idle":"2024-08-22T15:42:18.294550Z","shell.execute_reply.started":"2024-08-22T15:42:18.283770Z","shell.execute_reply":"2024-08-22T15:42:18.293631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = MyDataset(train)\nval_data = MyDataset(val)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.295702Z","iopub.execute_input":"2024-08-22T15:42:18.296441Z","iopub.status.idle":"2024-08-22T15:42:18.534512Z","shell.execute_reply.started":"2024-08-22T15:42:18.296408Z","shell.execute_reply":"2024-08-22T15:42:18.533495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_loader = DataLoader(train_data, batch_size = batch_size, shuffle = True)\nval_loader = DataLoader(val_data, batch_size = batch_size, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.535744Z","iopub.execute_input":"2024-08-22T15:42:18.536037Z","iopub.status.idle":"2024-08-22T15:42:18.541231Z","shell.execute_reply.started":"2024-08-22T15:42:18.536010Z","shell.execute_reply":"2024-08-22T15:42:18.540139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's first code a simple Convolution model inspired from LeNet architecture ","metadata":{}},{"cell_type":"code","source":"class SimpleModel(nn.Module):\n    def __init__(self, in_channels):\n        super().__init__()\n        self.layer1 = nn.Conv2d(in_channels, 6, 5) \n        self.layer2 = nn.Conv2d(6, 18, 3) # To calculate convout we use convout = (in_shape - num_filters + 1)*out_channels\n        self.fc1  = nn.Linear(5 * 5 * 18,120)\n        self.fc2 = nn.Linear(120, 30)\n        self.fc3 = nn.Linear(30, 10)\n        self.history = {}\n        \n    def forward(self, inputs):\n        x = inputs.permute(0, 3, 1, 2) # N, H, W, C ----> N, C, H, W where\n                                       #N:Batch_size, H, W : height and width, C:Channels\n        x = self.layer1(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2)\n        x = self.layer2(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2)\n        x = x.reshape(-1, self.num_flat_features(x))\n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        x = F.relu(x)\n        x = self.fc3(x)\n        x = F.softmax(x, dim=1)\n\n        return x\n       \n        \n    def num_flat_features(self, x): # Calculates the dimention of the convolution's output (after max pool)\n        size = x.size()[1:]  # all dimensions except the batch dimension\n        num_features = 1\n        for s in size:\n            num_features *= s\n        return num_features\n        ","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.542467Z","iopub.execute_input":"2024-08-22T15:42:18.542760Z","iopub.status.idle":"2024-08-22T15:42:18.553429Z","shell.execute_reply.started":"2024-08-22T15:42:18.542704Z","shell.execute_reply":"2024-08-22T15:42:18.552519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"simpleModel =  SimpleModel(1)\nsimpleModel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.556620Z","iopub.execute_input":"2024-08-22T15:42:18.556866Z","iopub.status.idle":"2024-08-22T15:42:18.570555Z","shell.execute_reply.started":"2024-08-22T15:42:18.556844Z","shell.execute_reply":"2024-08-22T15:42:18.569713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code the training function and a function to plot loss/accuracy per epoch evolution","metadata":{}},{"cell_type":"code","source":"phases = [\"train\", \"val\"]\nloader_dict = {\"train\" : training_loader, \"val\" : val_loader}\ndataset_dict = {\"train\" : train_data, \"val\" : val_data}\nnum_batch_dict = {\"train\" : len(train_data) // batch_size, \"val\" : len(val_data) // batch_size}","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.571651Z","iopub.execute_input":"2024-08-22T15:42:18.571974Z","iopub.status.idle":"2024-08-22T15:42:18.584769Z","shell.execute_reply.started":"2024-08-22T15:42:18.571942Z","shell.execute_reply":"2024-08-22T15:42:18.583911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_acc(n_epochs, history):\n    fig, axs = plt.subplots(2, 2, sharex = True)\n    x_axis = np.linspace(1, n_epochs, num = n_epochs)\n    axs[0, 0].plot(x_axis, history[\"train_loss\"])\n    axs[0, 0].set_title(f'train loss per epoch')\n    axs[0, 1].plot(x_axis, history[\"train_acc\"])\n    axs[0, 1].set_title(f'train accuracy per epoch')\n    axs[1, 0].plot(x_axis, history[\"val_loss\"])\n    axs[1, 0].set_title(f'validation loss per epoch')\n    axs[1, 1].plot(x_axis, history[\"val_acc\"])\n    axs[1, 1].set_title(f'validation accuracy per epoch')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.585845Z","iopub.execute_input":"2024-08-22T15:42:18.586163Z","iopub.status.idle":"2024-08-22T15:42:18.594289Z","shell.execute_reply.started":"2024-08-22T15:42:18.586132Z","shell.execute_reply":"2024-08-22T15:42:18.593387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, n_epochs, lr = 1e-2, optimizer = \"Adam\", plot = True):\n    history = {\n        \"train_loss\" : [],\n        \"train_acc\" : [],\n        \"val_loss\" : [],\n        \"val_acc\" : []\n    }\n    running_loss = 0\n    correct = 0\n    loss_fn = torch.nn.CrossEntropyLoss()\n    if optimizer == \"Adam\":\n        optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n    else : \n        optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n    model = model.to(device)\n    for epoch in tqdm(range(n_epochs)):\n        for phase in phases:\n            if phase == \"train\":\n                model.train()\n            if phase == \"val\":\n                model.eval()\n            for i, data in enumerate(loader_dict[phase]):  \n                inputs, labels = data\n                inputs, labels = inputs.to(device), labels.to(device)\n            \n                optimizer.zero_grad()\n                output = model(inputs)\n                loss = loss_fn(output, labels)\n                \n                if phase == \"train\":\n                    loss.backward()\n                    optimizer.step()\n                  \n                running_loss += loss.item() * len(inputs)\n                correct += (output.argmax(axis=1) == labels).sum()\n               \n        \n                if (i+1) % num_batch_dict[phase] == 0:\n                    avg_loss = running_loss / len(dataset_dict[phase])\n                    acc = correct / len(dataset_dict[phase])\n                    history[phase + \"_loss\"].append(avg_loss)\n                    history[phase + \"_acc\"].append(acc.item())\n                    running_loss = 0\n                    correct = 0\n    if plot:\n        plot_loss_acc(n_epochs, history)\n    return history\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.595348Z","iopub.execute_input":"2024-08-22T15:42:18.595640Z","iopub.status.idle":"2024-08-22T15:42:18.609038Z","shell.execute_reply.started":"2024-08-22T15:42:18.595617Z","shell.execute_reply":"2024-08-22T15:42:18.608085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"simpleModel =  SimpleModel(1)\nhistory1 = train(simpleModel, 35, lr = 1e-3)\nhistory2 = train(simpleModel, 10, lr=1e-3, optimizer=\"SGD\") #Usse the SGD optim to furhter fine tune model params","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:42:18.610026Z","iopub.execute_input":"2024-08-22T15:42:18.610309Z","iopub.status.idle":"2024-08-22T15:43:31.456109Z","shell.execute_reply.started":"2024-08-22T15:42:18.610286Z","shell.execute_reply":"2024-08-22T15:43:31.455160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ntest_data = MyDataset(test, test=True)\ntest_loader = DataLoader(test_data, batch_size = batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:43:31.457173Z","iopub.execute_input":"2024-08-22T15:43:31.457444Z","iopub.status.idle":"2024-08-22T15:43:33.154214Z","shell.execute_reply.started":"2024-08-22T15:43:31.457421Z","shell.execute_reply":"2024-08-22T15:43:33.153399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model):\n    predictions = []\n    model.eval()\n    for i, data in enumerate(test_loader):\n        inputs, _ = data\n        inputs = inputs.to(device)\n        with torch.no_grad():\n            output = model(inputs)\n            output = output.argmax(axis = 1)\n            output = output.cpu().numpy()\n        for out in output:\n            predictions.append(out)\n    return predictions\n    ","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:45:48.917602Z","iopub.execute_input":"2024-08-22T15:45:48.917964Z","iopub.status.idle":"2024-08-22T15:45:48.924598Z","shell.execute_reply.started":"2024-08-22T15:45:48.917934Z","shell.execute_reply":"2024-08-22T15:45:48.923627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = predict(simpleModel)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:45:51.877150Z","iopub.execute_input":"2024-08-22T15:45:51.877887Z","iopub.status.idle":"2024-08-22T15:45:52.429963Z","shell.execute_reply.started":"2024-08-22T15:45:51.877853Z","shell.execute_reply":"2024-08-22T15:45:52.429060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:45:54.768463Z","iopub.execute_input":"2024-08-22T15:45:54.768829Z","iopub.status.idle":"2024-08-22T15:45:54.786890Z","shell.execute_reply.started":"2024-08-22T15:45:54.768801Z","shell.execute_reply":"2024-08-22T15:45:54.785960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[\"Label\"] = predictions\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:45:57.507783Z","iopub.execute_input":"2024-08-22T15:45:57.508525Z","iopub.status.idle":"2024-08-22T15:45:57.632422Z","shell.execute_reply.started":"2024-08-22T15:45:57.508492Z","shell.execute_reply":"2024-08-22T15:45:57.631762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DeeperModel(nn.Module):\n    def __init__(self, in_channels):\n        super().__init__()\n        self.layer1 = nn.Conv2d(in_channels, 24, 5) # Conv2d(num_of_input_channels, num_of_output_chan, kernel size)\n        self.layer2 = nn.Conv2d(24, 18, 3)\n        # To calculate convout we use convout = (in_shpae - numf_filters + 1)*out_channels, so im_size_out = ((28 - 5 + 1)/2 - 3 + 1)/2 = 5\n        self.fc1  = nn.Linear(5 * 5 * 18,120)\n        self.fc2 = nn.Linear(120, 180)\n        self.fc3 = nn.Linear(180, 90)\n        self.fc4 = nn.Linear(90, 40)\n        self.fc5 = nn.Linear(40, 10)\n        \n        \n    def forward(self, inputs):\n        x = inputs.permute(0, 3, 1, 2) # N, H, W, C ----> N, C, H, W\n        x = self.layer1(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2)\n        x = self.layer2(x) \n        x = F.relu(x)\n        x = F.max_pool2d(x, 2)\n        x = x.reshape(-1, self.num_flat_features(x))\n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        x = F.relu(x)\n        x = self.fc3(x)\n        x = F.relu(x) + x\n        x = self.fc4(x)\n        x = F.relu(x) + x\n        x = self.fc5(x) \n        x = F.softmax(x, dim=1)\n\n        return x\n       \n        \n    def num_flat_features(self, x):\n        size = x.size()[1:]  # all dimensions except the batch dimension\n        num_features = 1\n        for s in size:\n            num_features *= s\n        return num_features\n        ","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:46:26.209925Z","iopub.execute_input":"2024-08-22T15:46:26.210842Z","iopub.status.idle":"2024-08-22T15:46:26.221725Z","shell.execute_reply.started":"2024-08-22T15:46:26.210807Z","shell.execute_reply":"2024-08-22T15:46:26.220823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Deepermodel = DeeperModel(1)\nhistory = train(Deepermodel, 30, lr=1e-3)\nhistory2 = train(Deepermodel, 10, lr=5e-3, optimizer=\"SGD\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:46:28.786008Z","iopub.execute_input":"2024-08-22T15:46:28.786387Z","iopub.status.idle":"2024-08-22T15:46:32.367539Z","shell.execute_reply.started":"2024-08-22T15:46:28.786342Z","shell.execute_reply":"2024-08-22T15:46:32.365827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = predict(Deepermodel)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:43:33.650685Z","iopub.status.idle":"2024-08-22T15:43:33.651024Z","shell.execute_reply.started":"2024-08-22T15:43:33.650861Z","shell.execute_reply":"2024-08-22T15:43:33.650875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\nsubmission[\"Label\"] = predictions\n#submission.to_csv('Deepersubmission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-22T15:43:33.652092Z","iopub.status.idle":"2024-08-22T15:43:33.652424Z","shell.execute_reply.started":"2024-08-22T15:43:33.652239Z","shell.execute_reply":"2024-08-22T15:43:33.652257Z"},"trusted":true},"execution_count":null,"outputs":[]}]}