{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport matplotlib.pyplot as plt\nfrom tqdm import trange","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:46:35.183853Z","iopub.execute_input":"2022-12-14T11:46:35.184438Z","iopub.status.idle":"2022-12-14T11:46:35.191895Z","shell.execute_reply.started":"2022-12-14T11:46:35.184394Z","shell.execute_reply":"2022-12-14T11:46:35.190442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('PyTorch', torch.__version__, 'Device:', device)\n\nrandom_seed = 42\ntorch.manual_seed(random_seed)\n\nvalidation_set_ratio = 0.1\n\nbatch_size = 320\nepoch_number = 100\ntensor_dtype = torch.float32","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:46:35.194687Z","iopub.execute_input":"2022-12-14T11:46:35.195254Z","iopub.status.idle":"2022-12-14T11:46:35.205888Z","shell.execute_reply.started":"2022-12-14T11:46:35.195208Z","shell.execute_reply":"2022-12-14T11:46:35.204849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.environ.get('KAGGLE_KERNEL_RUN_TYPE'):\n    input_path = '/kaggle/input/digit-recognizer'\n    output_path = ''\nelse:\n    input_path = 'data/digit-recognizer'\n    output_path = 'data/digit-recognizer'\n\ntrain_validation_df = pd.read_csv(os.path.join(input_path, 'train.csv'))\ntest_df = pd.read_csv(os.path.join(input_path, 'test.csv'))\nsubmission_df = pd.read_csv(os.path.join(input_path, 'sample_submission.csv'))\n\ntrain_df, validation_df = np.split(\n    train_validation_df.sample(frac=1, random_state=random_seed),\n    [int((1 - validation_set_ratio) * len(train_validation_df))])\n\nX_train = torch.tensor(train_df.iloc[:, 1:].values.reshape(-1, 28, 28), dtype=tensor_dtype)\ny_train = torch.tensor(train_df.iloc[:, 0].values)\nX_validation = torch.tensor(validation_df.iloc[:, 1:].values.reshape(-1, 28, 28), dtype=tensor_dtype)\ny_validation = torch.tensor(validation_df.iloc[:, 0].values)\nX_test = torch.tensor(test_df.values.reshape(-1, 28, 28), dtype=tensor_dtype)\n\ntrain_dataset = torch.utils.data.TensorDataset(X_train, y_train)\nvalidation_dataset = torch.utils.data.TensorDataset(X_validation, y_validation)\ntest_dataset = torch.utils.data.TensorDataset(X_test)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, pin_memory=True, shuffle=True)\nvalidation_loader = torch.utils.data.DataLoader(validation_dataset, batch_size=batch_size, pin_memory=True, shuffle=False)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, pin_memory=True, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:46:35.207397Z","iopub.execute_input":"2022-12-14T11:46:35.208629Z","iopub.status.idle":"2022-12-14T11:46:39.654201Z","shell.execute_reply.started":"2022-12-14T11:46:35.208587Z","shell.execute_reply":"2022-12-14T11:46:39.653126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 5)\nfig.suptitle('Sample Images')\nfor i in range(0, axes.shape[0]):\n    for j in range(0, axes.shape[1]):\n        k = i * axes.shape[1] + j\n        axes[i][j].imshow(X_train[k], cmap='gray')\n        axes[i][j].title.set_text(round(y_train[k].item()))\n        axes[i][j].set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:46:39.657343Z","iopub.execute_input":"2022-12-14T11:46:39.657829Z","iopub.status.idle":"2022-12-14T11:46:40.020721Z","shell.execute_reply.started":"2022-12-14T11:46:39.657783Z","shell.execute_reply":"2022-12-14T11:46:40.019499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MLP_128_64(nn.Module):\n    def __init__(self):\n        super(MLP_128_64, self).__init__()\n        self.linear1 = nn.Linear(28 * 28, 128, dtype=tensor_dtype)\n        self.linear2 = nn.Linear(128, 64, dtype=tensor_dtype)\n        self.linear3 = nn.Linear(64, 10, dtype=tensor_dtype)\n\n    def forward(self, x):\n        x = x.view(-1, 28 * 28)\n        x = F.relu(self.linear1(x))\n        x = F.relu(self.linear2(x))\n        x = F.log_softmax(self.linear3(x), dim=1)\n        return x\n\n    def reset_parameters(self):\n        self.linear1.reset_parameters()\n        self.linear2.reset_parameters()\n        self.linear3.reset_parameters()\n\ndef train(model, data_loader, criterion, optimizer):\n    model.train()\n    total_loss, correct = 0.0, 0\n\n    for data, target in data_loader:\n        data = data.to(device)\n        target = target.to(device)\n        optimizer.zero_grad() \n\n        output = model(data)\n        loss = criterion(output, target)\n        total_loss += loss.data.item()\n\n        pred = output.data.max(1)[1]\n        correct += pred.eq(target.data).sum().data.item()\n\n        loss.backward()\n        optimizer.step()\n\n    return total_loss, correct\n\ndef validate(model, data_loader, criterion):\n    model.eval()\n    total_loss, correct = 0.0, 0\n\n    for data, target in data_loader:\n        data = data.to(device)\n        target = target.to(device)\n\n        output = model(data)\n        loss = criterion(output, target)\n\n        total_loss += loss.data.item()\n        pred = output.data.max(1)[1]\n        correct += pred.eq(target.data).sum().data.item()\n\n    return total_loss, correct","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:46:40.022276Z","iopub.execute_input":"2022-12-14T11:46:40.022735Z","iopub.status.idle":"2022-12-14T11:46:40.038116Z","shell.execute_reply.started":"2022-12-14T11:46:40.022687Z","shell.execute_reply":"2022-12-14T11:46:40.036918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MLP_128_64().to(device)\n\noptimizer = torch.optim.SGD(model.parameters(), lr=0.001)\ncriterion = nn.CrossEntropyLoss()\n\ntrain_loss, train_accuracy = [], []\nvalidation_loss, validation_accuracy = [], []\n\nmodel.reset_parameters()\n\nprogress_bar = trange(epoch_number, desc='Training model', unit='epochs')\nfor epoch in progress_bar:\n    loss, accuracy = train(model, train_loader, criterion, optimizer)\n    train_loss.append(loss / len(train_loader))\n    train_accuracy.append(accuracy / len(train_loader.dataset))\n\n    loss, accuracy = validate(model, validation_loader, criterion)\n    validation_loss.append(loss / len(validation_loader))\n    validation_accuracy.append(accuracy / len(validation_loader.dataset))\n\n    progress_bar.set_postfix(\n        train_loss=train_loss[-1], train_accuracy=train_accuracy[-1],\n        validation_loss=validation_loss[-1], validation_accuracy=validation_accuracy[-1])","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:46:40.039892Z","iopub.execute_input":"2022-12-14T11:46:40.040423Z","iopub.status.idle":"2022-12-14T11:48:42.883374Z","shell.execute_reply.started":"2022-12-14T11:46:40.040381Z","shell.execute_reply":"2022-12-14T11:48:42.882268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5, 3))\nplt.plot(np.arange(1, epoch_number + 1), train_loss)\nplt.plot(np.arange(1, epoch_number + 1), validation_loss)\nplt.title('Train and Validation Loss')\n\nplt.figure(figsize=(5, 3))\nplt.plot(np.arange(1, epoch_number + 1), train_accuracy)\nplt.plot(np.arange(1, epoch_number + 1), validation_accuracy)\nplt.title('Train and Validation Accuracy')","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:48:42.886164Z","iopub.execute_input":"2022-12-14T11:48:42.887025Z","iopub.status.idle":"2022-12-14T11:48:43.184915Z","shell.execute_reply.started":"2022-12-14T11:48:42.886979Z","shell.execute_reply":"2022-12-14T11:48:43.183743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    fig, axes = plt.subplots(3, 10)\n    fig.suptitle('Sample Predictions')\n    for i in range(0, axes.shape[0]):\n        for j in range(0, axes.shape[1]):\n            k = i * axes.shape[1] + j\n            axes[i][j].imshow(X_test[k], cmap='gray')\n            axes[i][j].title.set_text(torch.max(model(X_test[k].to(device)), 1).indices.item())\n            axes[i][j].set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:48:43.186833Z","iopub.execute_input":"2022-12-14T11:48:43.187296Z","iopub.status.idle":"2022-12-14T11:48:44.404957Z","shell.execute_reply.started":"2022-12-14T11:48:43.187252Z","shell.execute_reply":"2022-12-14T11:48:44.403676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = pd.Series(dtype=np.int32)\nwith torch.no_grad():\n    for [data] in test_loader:\n        data = data.to(device)\n        output = model(data)\n        pred = output.data.max(1)[1]\n        y_test = y_test.append(pd.Series(pred.cpu().numpy()))\n\nsubmission_df['Label'] = y_test.to_numpy()\nprint(submission_df)\nsubmission_df.to_csv(os.path.join(output_path, 'submission.csv'), index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T11:48:44.406224Z","iopub.execute_input":"2022-12-14T11:48:44.406557Z","iopub.status.idle":"2022-12-14T11:48:44.717726Z","shell.execute_reply.started":"2022-12-14T11:48:44.406526Z","shell.execute_reply":"2022-12-14T11:48:44.716916Z"},"trusted":true},"execution_count":null,"outputs":[]}]}