{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport matplotlib.pyplot as plt\nfrom tqdm import trange","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:46:30.360564Z","iopub.execute_input":"2022-12-16T10:46:30.361574Z","iopub.status.idle":"2022-12-16T10:46:32.386371Z","shell.execute_reply.started":"2022-12-16T10:46:30.361457Z","shell.execute_reply":"2022-12-16T10:46:32.385174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('PyTorch', torch.__version__, 'Device:', device)\n\nrandom_seed = 42\ntorch.manual_seed(random_seed)\n\nvalidation_set_ratio = 0.1\n\nbatch_size = 320\nepoch_number = 20\ntensor_dtype = torch.float32","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:46:32.388827Z","iopub.execute_input":"2022-12-16T10:46:32.390007Z","iopub.status.idle":"2022-12-16T10:46:32.403675Z","shell.execute_reply.started":"2022-12-16T10:46:32.389954Z","shell.execute_reply":"2022-12-16T10:46:32.402254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.environ.get('KAGGLE_KERNEL_RUN_TYPE'):\n    input_path = '/kaggle/input/digit-recognizer'\n    output_path = ''\nelse:\n    input_path = 'data/digit-recognizer'\n    output_path = 'results/digit-recognizer'\n\ntrain_validation_df = pd.read_csv(os.path.join(input_path, 'train.csv'))\ntest_df = pd.read_csv(os.path.join(input_path, 'test.csv'))\nsubmission_df = pd.read_csv(os.path.join(input_path, 'sample_submission.csv'))\n\ntrain_df, validation_df = np.split(\n    train_validation_df.sample(frac=1, random_state=random_seed),\n    [int((1 - validation_set_ratio) * len(train_validation_df))])\n\nX_train = torch.tensor(train_df.iloc[:, 1:].values.reshape(-1, 28, 28), dtype=tensor_dtype)\ny_train = torch.tensor(train_df.iloc[:, 0].values)\nX_validation = torch.tensor(validation_df.iloc[:, 1:].values.reshape(-1, 28, 28), dtype=tensor_dtype)\ny_validation = torch.tensor(validation_df.iloc[:, 0].values)\nX_test = torch.tensor(test_df.values.reshape(-1, 28, 28), dtype=tensor_dtype)\n\ntrain_dataset = torch.utils.data.TensorDataset(X_train, y_train)\nvalidation_dataset = torch.utils.data.TensorDataset(X_validation, y_validation)\ntest_dataset = torch.utils.data.TensorDataset(X_test)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, pin_memory=True, shuffle=True)\nvalidation_loader = torch.utils.data.DataLoader(validation_dataset, batch_size=batch_size, pin_memory=True, shuffle=False)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, pin_memory=True, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:46:32.405059Z","iopub.execute_input":"2022-12-16T10:46:32.405425Z","iopub.status.idle":"2022-12-16T10:46:39.774708Z","shell.execute_reply.started":"2022-12-16T10:46:32.405394Z","shell.execute_reply":"2022-12-16T10:46:39.773750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 5)\nfig.suptitle('Sample Images')\nfor i in range(0, axes.shape[0]):\n    for j in range(0, axes.shape[1]):\n        k = i * axes.shape[1] + j\n        axes[i][j].imshow(X_train[k], cmap='gray')\n        axes[i][j].title.set_text(round(y_train[k].item()))\n        axes[i][j].set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:46:39.777033Z","iopub.execute_input":"2022-12-16T10:46:39.778200Z","iopub.status.idle":"2022-12-16T10:46:40.270298Z","shell.execute_reply.started":"2022-12-16T10:46:39.778162Z","shell.execute_reply":"2022-12-16T10:46:40.268944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_conv_size = 32\nout_conv_size = 16\nout_conv_width = out_conv_height = 5\n\nclass CNN_32_16(nn.Module):\n    def __init__(self):\n        super(CNN_32_16, self).__init__()\n        self.conv1 = nn.Conv2d(1, in_conv_size, kernel_size=3, dtype=tensor_dtype)\n        self.conv2 = nn.Conv2d(in_conv_size, out_conv_size, kernel_size=3, dtype=tensor_dtype)\n        self.linear1 = nn.Linear(out_conv_size * out_conv_width * out_conv_height, 10, dtype=tensor_dtype)\n\n    def forward(self, x):\n        if x.dim() == 3:\n            x = x[:, None, :, :]\n        else:\n            x = x[None, :, :]\n        x = x / 255\n        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n        self.conv1_output = x\n        x = F.relu(F.max_pool2d(self.conv2(x), 2))\n        self.conv2_output = x\n        x = x.view(-1, out_conv_size * out_conv_width * out_conv_height)\n        x = F.log_softmax(self.linear1(x), dim=1)\n        return x\n\n    def reset_parameters(self):\n        self.conv1.reset_parameters()\n        self.conv1.reset_parameters()\n        self.linear1.reset_parameters()\n\ndef train(model, data_loader, criterion, optimizer):\n    model.train()\n    total_loss, correct = 0.0, 0\n\n    for data, target in data_loader:\n        data = data.to(device)\n        target = target.to(device)\n        optimizer.zero_grad() \n\n        output = model(data)\n        loss = criterion(output, target)\n        total_loss += loss.data.item()\n\n        pred = output.data.max(1)[1]\n        correct += pred.eq(target.data).sum().data.item()\n\n        loss.backward()\n        optimizer.step()\n\n    return total_loss, correct\n\ndef validate(model, data_loader, criterion):\n    model.eval()\n    total_loss, correct = 0.0, 0\n\n    for data, target in data_loader:\n        data = data.to(device)\n        target = target.to(device)\n\n        output = model(data)\n        loss = criterion(output, target)\n\n        total_loss += loss.data.item()\n        pred = output.data.max(1)[1]\n        correct += pred.eq(target.data).sum().data.item()\n\n    return total_loss, correct","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:46:40.272043Z","iopub.execute_input":"2022-12-16T10:46:40.272438Z","iopub.status.idle":"2022-12-16T10:46:40.290841Z","shell.execute_reply.started":"2022-12-16T10:46:40.272404Z","shell.execute_reply":"2022-12-16T10:46:40.289580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN_32_16().to(device)\n\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\ncriterion = nn.CrossEntropyLoss()\n\ntrain_loss, train_accuracy = [], []\nvalidation_loss, validation_accuracy = [], []\n\nmodel.reset_parameters()\n\nprogress_bar = trange(epoch_number, desc='Training model', unit='epochs')\nfor epoch in progress_bar:\n    loss, accuracy = train(model, train_loader, criterion, optimizer)\n    train_loss.append(loss / len(train_loader))\n    train_accuracy.append(accuracy / len(train_loader.dataset))\n\n    loss, accuracy = validate(model, validation_loader, criterion)\n    validation_loss.append(loss / len(validation_loader))\n    validation_accuracy.append(accuracy / len(validation_loader.dataset))\n\n    progress_bar.set_postfix(\n        train_loss=train_loss[-1], train_accuracy=train_accuracy[-1],\n        validation_loss=validation_loss[-1], validation_accuracy=validation_accuracy[-1])","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:46:40.292504Z","iopub.execute_input":"2022-12-16T10:46:40.293221Z","iopub.status.idle":"2022-12-16T10:51:48.941795Z","shell.execute_reply.started":"2022-12-16T10:46:40.293168Z","shell.execute_reply":"2022-12-16T10:51:48.940528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5, 3))\nplt.plot(np.arange(1, epoch_number + 1), train_loss)\nplt.plot(np.arange(1, epoch_number + 1), validation_loss)\nplt.title('Train and Validation Loss')\n\nplt.figure(figsize=(5, 3))\nplt.plot(np.arange(1, epoch_number + 1), train_accuracy)\nplt.plot(np.arange(1, epoch_number + 1), validation_accuracy)\nplt.title('Train and Validation Accuracy')","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:51:48.943549Z","iopub.execute_input":"2022-12-16T10:51:48.944189Z","iopub.status.idle":"2022-12-16T10:51:49.395258Z","shell.execute_reply.started":"2022-12-16T10:51:48.944153Z","shell.execute_reply":"2022-12-16T10:51:49.394346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cells = model.conv1.weight.shape[0]\nrows = round(np.sqrt(cells))\ncols = round(np.ceil(cells / rows))\n\nfig, axes = plt.subplots(rows, cols)\nfig.suptitle('Conv1 Weights')\nfor i in range(0, rows):\n    for j in range(0, cols):\n        k = i * cols + j\n        if rows > 1:\n            ax = axes[i][j] if cols > 1 else axes[i]\n        else:\n            ax = axes[j] if cols > 1 else axes\n        if k < cells:\n            ax.imshow(model.conv1.weight[k, 0].cpu().detach().numpy(), cmap='gray')\n        ax.set_axis_off()\n\nfor m in range(0, 3):\n    with torch.no_grad():\n        model(X_test[m][None, :, :].to(device))\n        img = model.conv1_output[0].cpu().detach().numpy()\n        fig, axes = plt.subplots(rows, cols)\n        fig.suptitle('Conv1 Output')\n        for i in range(0, rows):\n            for j in range(0, cols):\n                k = i * cols + j\n                if rows > 1:\n                    ax = axes[i][j] if cols > 1 else axes[i]\n                else:\n                    ax = axes[j] if cols > 1 else axes\n                if k < cells:\n                    ax.imshow(img[k], cmap='gray')\n                ax.set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:51:49.396472Z","iopub.execute_input":"2022-12-16T10:51:49.397474Z","iopub.status.idle":"2022-12-16T10:51:54.749023Z","shell.execute_reply.started":"2022-12-16T10:51:49.397429Z","shell.execute_reply":"2022-12-16T10:51:54.747980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = model.conv2.weight.shape[0]\ncols = model.conv2.weight.shape[1]\n\nfig, axes = plt.subplots(rows, cols)\nfig.set_figwidth(15)\nfig.suptitle('Conv2 Weights')\nfor i in range(0, rows):\n    for j in range(0, cols):\n        if rows > 1:\n            ax = axes[i][j] if cols > 1 else axes[i]\n        else:\n            ax = axes[j] if cols > 1 else axes\n        ax.imshow(model.conv2.weight[i, j].cpu().detach().numpy(), cmap='gray')\n        ax.set_axis_off()\n\nfig, axes = plt.subplots(1, cols)\nfig.set_figwidth(15)\n# fig.suptitle('Sample Images')\nfor j in range(0, cols):\n    ax = axes[j] if cols > 1 else axes\n    ax.imshow(X_test[j], cmap='gray')\n    ax.set_axis_off()\n\nfig, axes = plt.subplots(rows, cols)\nfig.set_figwidth(15)\nfig.suptitle('Conv2 Output')\nfor j in range(0, cols):\n    with torch.no_grad():\n        model(X_test[j][None, :, :].to(device))\n        img = model.conv2_output[0].cpu().detach().numpy()\n        for i in range(0, rows):\n            if rows > 1:\n                ax = axes[i][j] if cols > 1 else axes[i]\n            else:\n                ax = axes[j] if cols > 1 else axes\n            ax.imshow(img[i], cmap='gray')\n            ax.set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:51:54.750786Z","iopub.execute_input":"2022-12-16T10:51:54.752076Z","iopub.status.idle":"2022-12-16T10:52:33.873157Z","shell.execute_reply.started":"2022-12-16T10:51:54.752020Z","shell.execute_reply":"2022-12-16T10:52:33.872036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    fig, axes = plt.subplots(3, 10)\n    fig.suptitle('Sample Predictions')\n    for i in range(0, axes.shape[0]):\n        for j in range(0, axes.shape[1]):\n            k = i * axes.shape[1] + j\n            axes[i][j].imshow(X_test[k], cmap='gray')\n            axes[i][j].title.set_text(torch.max(model(X_test[k].to(device)), 1).indices.item())\n            axes[i][j].set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:52:33.875517Z","iopub.execute_input":"2022-12-16T10:52:33.875860Z","iopub.status.idle":"2022-12-16T10:52:34.978811Z","shell.execute_reply.started":"2022-12-16T10:52:33.875829Z","shell.execute_reply":"2022-12-16T10:52:34.977349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = pd.Series(dtype=np.int32)\nwith torch.no_grad():\n    for [data] in test_loader:\n        data = data.to(device)\n        output = model(data)\n        pred = output.data.max(1)[1]\n        y_test = y_test.append(pd.Series(pred.cpu().numpy()))\n\nsubmission_df['Label'] = y_test.to_numpy()\nprint(submission_df)\nsubmission_df.to_csv(os.path.join(output_path, 'submission.csv'), index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:52:34.980295Z","iopub.execute_input":"2022-12-16T10:52:34.980637Z","iopub.status.idle":"2022-12-16T10:52:39.517488Z","shell.execute_reply.started":"2022-12-16T10:52:34.980607Z","shell.execute_reply":"2022-12-16T10:52:39.516288Z"},"trusted":true},"execution_count":null,"outputs":[]}]}