{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer CNN","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split \nfrom sklearn.preprocessing import OneHotEncoder\nimport torch\nfrom torch import nn\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import DataLoader, Dataset, IterableDataset\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:20:52.752162Z","iopub.execute_input":"2023-02-09T17:20:52.752719Z","iopub.status.idle":"2023-02-09T17:20:52.760898Z","shell.execute_reply.started":"2023-02-09T17:20:52.752682Z","shell.execute_reply":"2023-02-09T17:20:52.759724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:20:56.151946Z","iopub.execute_input":"2023-02-09T17:20:56.152309Z","iopub.status.idle":"2023-02-09T17:20:56.221367Z","shell.execute_reply.started":"2023-02-09T17:20:56.152277Z","shell.execute_reply":"2023-02-09T17:20:56.220380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install torch-summary","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:41:34.391301Z","iopub.execute_input":"2023-02-09T17:41:34.391693Z","iopub.status.idle":"2023-02-09T17:41:45.295529Z","shell.execute_reply.started":"2023-02-09T17:41:34.391634Z","shell.execute_reply":"2023-02-09T17:41:45.294158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary #packet fo testing model","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:41:45.297978Z","iopub.execute_input":"2023-02-09T17:41:45.299228Z","iopub.status.idle":"2023-02-09T17:41:45.312641Z","shell.execute_reply.started":"2023-02-09T17:41:45.299193Z","shell.execute_reply":"2023-02-09T17:41:45.311691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load data**","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:20:59.667490Z","iopub.execute_input":"2023-02-09T17:20:59.667862Z","iopub.status.idle":"2023-02-09T17:21:04.646237Z","shell.execute_reply.started":"2023-02-09T17:20:59.667833Z","shell.execute_reply":"2023-02-09T17:21:04.645235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:21:04.647969Z","iopub.execute_input":"2023-02-09T17:21:04.648333Z","iopub.status.idle":"2023-02-09T17:21:04.674973Z","shell.execute_reply.started":"2023-02-09T17:21:04.648298Z","shell.execute_reply":"2023-02-09T17:21:04.674157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset class for forming dataloader**","metadata":{}},{"cell_type":"code","source":"class MNIST_Dataset(Dataset):\n    def __init__(self, digits, labels):\n        self.digits = digits\n        self.labels = labels\n        \n    def __getitem__(self, idx):\n        label = self.labels[idx]\n        digit = self.digits[idx]\n        return (digit, label)\n    def __len__(self):\n        return len(self.labels)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:21:13.532840Z","iopub.execute_input":"2023-02-09T17:21:13.533461Z","iopub.status.idle":"2023-02-09T17:21:13.542786Z","shell.execute_reply.started":"2023-02-09T17:21:13.533427Z","shell.execute_reply":"2023-02-09T17:21:13.541611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Split data train and test**","metadata":{}},{"cell_type":"code","source":"def data_split_and_encode(df, ohe=False):\n    df = df / 255\n    X_train, X_test, y_train, y_test = train_test_split(df.drop(labels='label', axis=1).to_numpy(), \n                                                    df['label'], test_size=0.15)\n    dim_2d = X_train.shape[1]**0.5\n    if int(dim_2d) == dim_2d:\n        X_train = X_train.reshape(-1, 28, 28)\n        X_test = X_test.reshape(-1, 28, 28)\n    if ohe:\n        y_train = OneHotEncoder(sparse=False).fit_transform(y_train.to_numpy().reshape(-1,1))\n        y_test = OneHotEncoder(sparse=False).fit_transform(y_test.to_numpy().reshape(-1,1))\n\n    train_dataset = MNIST_Dataset(X_train, y_train)\n    test_dataset = MNIST_Dataset(X_test, y_test)\n    return train_dataset, test_dataset","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:25:30.309173Z","iopub.execute_input":"2023-02-09T17:25:30.309569Z","iopub.status.idle":"2023-02-09T17:25:30.317065Z","shell.execute_reply.started":"2023-02-09T17:25:30.309536Z","shell.execute_reply":"2023-02-09T17:25:30.315872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset,test_dataset = data_split_and_encode(train_data, True)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:25:31.815447Z","iopub.execute_input":"2023-02-09T17:25:31.815829Z","iopub.status.idle":"2023-02-09T17:25:32.393482Z","shell.execute_reply.started":"2023-02-09T17:25:31.815798Z","shell.execute_reply":"2023-02-09T17:25:32.392345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Create Pytorch dataloaders**","metadata":{}},{"cell_type":"code","source":"train_loader = DataLoader(train_dataset,batch_size=32)\ntest_loader = DataLoader(test_dataset,batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:25:34.147813Z","iopub.execute_input":"2023-02-09T17:25:34.148780Z","iopub.status.idle":"2023-02-09T17:25:34.154945Z","shell.execute_reply.started":"2023-02-09T17:25:34.148736Z","shell.execute_reply":"2023-02-09T17:25:34.153906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader.dataset[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Convolutional neural network model**","metadata":{}},{"cell_type":"code","source":"class Model_CNN(nn.Module):\n    def __init__(self, n_features, bias=True, batch_norm=False, dropout=False):\n        super().__init__()\n        self.history = {\n            'score': {\n                'train': [],\n                'test': []\n                }\n            }\n        self.bias = bias\n        self.bn = batch_norm\n        self.dropout = dropout\n        \n        self.conv1 = self.conv_layer(n_in=n_features, n_out=64) # 64*27*27\n        self.conv2 = self.conv_layer(64, 128, kernel_size=3) # 128*25*25\n        self.conv3 = self.conv_layer(128, 256, kernel_size=3) # 256*23*23\n        self.conv4 = self.conv_layer(256, 128, kernel_size=3) # 128*21*21\n        self.conv5 = self.conv_layer(128, 64) # 64*20*20\n        self.linear_1 = nn.Linear(in_features=64*20*20, out_features=10)\n        \n        \n    def conv_layer(self, n_in, n_out, kernel_size=2):\n        seq = [nn.Conv2d(in_channels=n_in, out_channels=n_out, kernel_size=kernel_size, bias=self.bias)]\n        if self.bn:\n            seq.append(nn.BatchNorm2d(n_out))\n        seq.append(nn.ReLU())\n        if self.dropout:\n            seq.append(nn.Dropout(0.5))\n        return nn.Sequential(*seq)\n        \n    def forward(self, x):\n        bs = x.shape[0]\n        x = x.view(bs, 1, x.shape[1], x.shape[2])\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n        x = self.conv5(x)\n        x = x.view(bs,-1)\n        x = self.linear_1(x)\n        x = nn.Softmax(dim=1)(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:47:09.010551Z","iopub.execute_input":"2023-02-09T17:47:09.011563Z","iopub.status.idle":"2023-02-09T17:47:09.024004Z","shell.execute_reply.started":"2023-02-09T17:47:09.011526Z","shell.execute_reply":"2023-02-09T17:47:09.022771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model_CNN(1, bias=False, batch_norm=True, dropout=True)\nsummary(model, (28,28))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:47:09.382731Z","iopub.execute_input":"2023-02-09T17:47:09.383563Z","iopub.status.idle":"2023-02-09T17:47:09.411470Z","shell.execute_reply.started":"2023-02-09T17:47:09.383519Z","shell.execute_reply":"2023-02-09T17:47:09.410479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Function Training model. In process show image and prediction**","metadata":{}},{"cell_type":"code","source":"def train_model(train_loader, test_loader, model, loss, opt, epochs=15):\n\n    for epoch in range(epochs):\n        print(f'Start epoch - {epoch+1}')\n        model.train()\n        i = 0\n        score_train = []\n        score_test = []\n        \n        for X_train, y_train in train_loader:\n            X_train = X_train.type(torch.float).to(device)\n            y_train = y_train.to(device)\n            opt.zero_grad()\n            pred = model(X_train)\n            _loss = loss(y_train, pred)\n            score = [torch.argmax(target) == torch.argmax(pred_d) for target, pred_d in zip(y_train, pred)]\n            score = [el.cpu().numpy() for el in score]\n            _loss.backward()\n            opt.step()\n            score_train.append(sum(score)/len(score))\n\n        with torch.no_grad():\n            model.eval()\n            for X_test, y_test in test_loader:\n                i += 1\n                X_test = X_test.type(torch.float).to(device)\n                y_test = y_test.to(device)\n                pred_test = model(X_test)\n                score = [torch.argmax(target) == torch.argmax(pred_d) for target, pred_d in zip(y_test, pred_test)]\n                score = [el.cpu().numpy() for el in score]\n                score_test.append(sum(score)/len(score))\n\n                if i%100 == 0:\n                    random_digit_1 = np.random.randint(0, X_test.shape[0])\n                    random_digit_2 = np.random.randint(0, X_test.shape[0])\n                    plt.subplot(1,2,1)\n                    plt.imshow(X_test[random_digit_1].cpu().numpy())\n                    plt.title(f'Predict {torch.argmax(pred_test[random_digit_1])}')\n                    plt.axis('off')\n                    plt.subplot(1,2,2)\n                    plt.imshow(X_test[random_digit_2].cpu().numpy())\n                    plt.title(f'Predict {torch.argmax(pred_test[random_digit_2])}')\n                    plt.axis('off')\n                    plt.show()\n            \n        model.history['score']['train'].append(sum(score_train)/len(score_train))\n        model.history['score']['test'].append(sum(score_test)/len(score_test))\n        \n        print(f\"Accuracy train: {model.history['score']['train']} -- Accuracy test: {model.history['score']['test']}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:47:12.131409Z","iopub.execute_input":"2023-02-09T17:47:12.132438Z","iopub.status.idle":"2023-02-09T17:47:12.147693Z","shell.execute_reply.started":"2023-02-09T17:47:12.132390Z","shell.execute_reply":"2023-02-09T17:47:12.146710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model_CNN(1, bias=False, batch_norm=True, dropout=False).to(device)\nloss = nn.CrossEntropyLoss()\nopt = torch.optim.Adam(params=model.parameters(), lr=0.0001)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:47:17.837803Z","iopub.execute_input":"2023-02-09T17:47:17.838356Z","iopub.status.idle":"2023-02-09T17:47:17.870199Z","shell.execute_reply.started":"2023-02-09T17:47:17.838313Z","shell.execute_reply":"2023-02-09T17:47:17.868437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(train_loader, test_loader, model, loss, opt, epochs=15) #Training process","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:47:28.273615Z","iopub.execute_input":"2023-02-09T17:47:28.274019Z","iopub.status.idle":"2023-02-09T17:50:34.599567Z","shell.execute_reply.started":"2023-02-09T17:47:28.273980Z","shell.execute_reply":"2023-02-09T17:50:34.598396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Plot training/test results by epoch**","metadata":{}},{"cell_type":"code","source":"plt.plot(model.history['score']['train'])\nplt.plot(model.history['score']['test'])","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:50:54.615161Z","iopub.execute_input":"2023-02-09T17:50:54.615525Z","iopub.status.idle":"2023-02-09T17:50:54.806868Z","shell.execute_reply.started":"2023-02-09T17:50:54.615494Z","shell.execute_reply":"2023-02-09T17:50:54.805796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = test_data / 255","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:34:52.902563Z","iopub.execute_input":"2023-02-09T17:34:52.903186Z","iopub.status.idle":"2023-02-09T17:34:52.984209Z","shell.execute_reply.started":"2023-02-09T17:34:52.903138Z","shell.execute_reply":"2023-02-09T17:34:52.983212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_sub = torch.tensor(test_data.to_numpy().reshape(-1, 28, 28)).type(torch.float).to(device) #Convert test_data to numpy","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:35:43.810170Z","iopub.execute_input":"2023-02-09T17:35:43.810564Z","iopub.status.idle":"2023-02-09T17:35:44.051472Z","shell.execute_reply.started":"2023-02-09T17:35:43.810529Z","shell.execute_reply":"2023-02-09T17:35:44.050414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Predict test data**","metadata":{}},{"cell_type":"code","source":"test_data_pred = []\nfor i in range(0, 28000, 50):\n    pred = model(test_data_sub[i:i+50])\n    test_data_pred.append(pred.detach().cpu().numpy())\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:35:46.166951Z","iopub.execute_input":"2023-02-09T17:35:46.167327Z","iopub.status.idle":"2023-02-09T17:35:48.330996Z","shell.execute_reply.started":"2023-02-09T17:35:46.167295Z","shell.execute_reply":"2023-02-09T17:35:48.329992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Preprocessing and save results**","metadata":{}},{"cell_type":"code","source":"test_data_pred = np.array(test_data_pred)\ntest_data_pred = test_data_pred.reshape(28000,10)\ntest_data_pred = test_data_pred.argmax(axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:35:48.333138Z","iopub.execute_input":"2023-02-09T17:35:48.333722Z","iopub.status.idle":"2023-02-09T17:35:48.342123Z","shell.execute_reply.started":"2023-02-09T17:35:48.333683Z","shell.execute_reply":"2023-02-09T17:35:48.340979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_pred_pd = pd.DataFrame(test_data_pred, columns=['Label'])\ntest_data_pred_pd.index.rename('ImageId', inplace=True)\ntest_data_pred_pd.index = test_data_pred_pd.index + 1","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:35:49.688268Z","iopub.execute_input":"2023-02-09T17:35:49.688615Z","iopub.status.idle":"2023-02-09T17:35:49.695073Z","shell.execute_reply.started":"2023-02-09T17:35:49.688585Z","shell.execute_reply":"2023-02-09T17:35:49.694057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_pred_pd.to_csv('/kaggle/working/submission2.csv', )","metadata":{"execution":{"iopub.status.busy":"2023-02-09T17:35:53.194958Z","iopub.execute_input":"2023-02-09T17:35:53.195321Z","iopub.status.idle":"2023-02-09T17:35:53.231020Z","shell.execute_reply.started":"2023-02-09T17:35:53.195291Z","shell.execute_reply":"2023-02-09T17:35:53.230116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}