{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-18T08:55:17.412308Z","iopub.execute_input":"2021-10-18T08:55:17.412666Z","iopub.status.idle":"2021-10-18T08:55:17.422707Z","shell.execute_reply.started":"2021-10-18T08:55:17.412625Z","shell.execute_reply":"2021-10-18T08:55:17.42178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nfrom torch import optim, nn\nfrom torchvision import transforms, models, datasets\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm import tqdm\nimport shutil \n\n%matplotlib inline","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-10-18T08:55:26.917218Z","iopub.execute_input":"2021-10-18T08:55:26.917587Z","iopub.status.idle":"2021-10-18T08:55:28.850262Z","shell.execute_reply.started":"2021-10-18T08:55:26.917556Z","shell.execute_reply":"2021-10-18T08:55:28.849509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(torch.cuda.is_available())\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:55:36.83625Z","iopub.execute_input":"2021-10-18T08:55:36.836653Z","iopub.status.idle":"2021-10-18T08:55:36.902385Z","shell.execute_reply.started":"2021-10-18T08:55:36.836619Z","shell.execute_reply":"2021-10-18T08:55:36.901369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data","metadata":{}},{"cell_type":"code","source":"path = \"/kaggle/input/digit-recognizer/\"\ntrain_df = pd.read_csv(path + 'train.csv')\ntest_df = pd.read_csv(path + 'test.csv')\ntrain_df.shape, test_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:55:56.517138Z","iopub.execute_input":"2021-10-18T08:55:56.517499Z","iopub.status.idle":"2021-10-18T08:56:01.61881Z","shell.execute_reply.started":"2021-10-18T08:55:56.517466Z","shell.execute_reply":"2021-10-18T08:56:01.61789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Convert data in float","metadata":{}},{"cell_type":"code","source":"train_classes = train_df['label'].values\ntrain_img = (train_df.iloc[:, 1:].values).astype('float32')\ntest_img = (test_df.iloc[:, :].values).astype('float32')\ntrain_img.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:56:04.639417Z","iopub.execute_input":"2021-10-18T08:56:04.639767Z","iopub.status.idle":"2021-10-18T08:56:04.750129Z","shell.execute_reply.started":"2021-10-18T08:56:04.639734Z","shell.execute_reply":"2021-10-18T08:56:04.749169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split data in 80% to train and 20% to val","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(\n                                    train_img, train_classes, stratify = train_classes,\n                                    test_size = 0.2, random_state = 42)\nX_train.shape, y_train.shape, X_test.shape, test_img.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:56:09.796202Z","iopub.execute_input":"2021-10-18T08:56:09.796587Z","iopub.status.idle":"2021-10-18T08:56:10.135116Z","shell.execute_reply.started":"2021-10-18T08:56:09.796553Z","shell.execute_reply":"2021-10-18T08:56:10.133962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = np.array(X_train).reshape(33600, 1, 28, 28)\nval = X_test.reshape(8400, 1, 28, 28)\ntest = test_img.reshape(28000, 1, 28, 28)\n\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:57:13.700414Z","iopub.execute_input":"2021-10-18T08:57:13.700764Z","iopub.status.idle":"2021-10-18T08:57:13.749153Z","shell.execute_reply.started":"2021-10-18T08:57:13.700727Z","shell.execute_reply":"2021-10-18T08:57:13.748014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualization data","metadata":{}},{"cell_type":"code","source":"for i in range(3):\n    plt.subplot(330 + (i + 1))\n    plt.imshow(train[i].squeeze(), cmap=plt.get_cmap('gray'))\n    plt.title(y_train[i])","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:57:23.719316Z","iopub.execute_input":"2021-10-18T08:57:23.719642Z","iopub.status.idle":"2021-10-18T08:57:24.014181Z","shell.execute_reply.started":"2021-10-18T08:57:23.719612Z","shell.execute_reply":"2021-10-18T08:57:24.013203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train\nx_train_tensor = torch.tensor(train)/255\ny_train_tensor = torch.tensor(y_train)\ntrain_tensor = torch.utils.data.TensorDataset(x_train_tensor, y_train_tensor)\n\n# val\nx_test_tensor = torch.tensor(val)/255\ny_test_tensor = torch.tensor(y_test)\nval_tensor = torch.utils.data.TensorDataset(x_test_tensor, y_test_tensor)\n\n# test\ntest_tensor = torch.tensor(test)/255","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:57:42.242563Z","iopub.execute_input":"2021-10-18T08:57:42.243015Z","iopub.status.idle":"2021-10-18T08:57:42.544159Z","shell.execute_reply.started":"2021-10-18T08:57:42.242974Z","shell.execute_reply":"2021-10-18T08:57:42.543152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(train_tensor, batch_size=64, num_workers= 4, shuffle=True)\nval_loader = torch.utils.data.DataLoader(val_tensor, batch_size = 64, num_workers=4, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(test_tensor, batch_size= 64, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-18T08:59:58.293295Z","iopub.execute_input":"2021-10-18T08:59:58.29366Z","iopub.status.idle":"2021-10-18T08:59:58.299297Z","shell.execute_reply.started":"2021-10-18T08:59:58.293628Z","shell.execute_reply":"2021-10-18T08:59:58.298409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Resnet model","metadata":{}},{"cell_type":"code","source":"class Neural(nn.Module):\n    def __init__(self, model, output):\n        super(Neural, self).__init__()\n        self.model = nn.Sequential(\n                nn.Conv2d(1, 64, kernel_size = (7,7), stride = (2, 2), padding = (3, 3), bias = False),\n                *(list(model.children())[1:-1]),\n                nn.AdaptiveMaxPool2d(output_size = 1),\n                nn.Flatten(),\n                nn.BatchNorm1d(512, eps=1e-05, momentum=0.1, track_running_stats = True, affine = True),\n                nn.Dropout(p = 0.25, inplace = True),\n                )\n        self.fc = nn.Sequential(\n                nn.Linear(in_features=512, out_features = 128, bias=True),\n                nn.BatchNorm1d(128, eps=1e-05, momentum=0.1, track_running_stats = True, affine = True),\n                nn.Dropout(p=0.3, inplace = True),\n                nn.ReLU(inplace = True),\n                nn.Linear(128, output, bias = True),\n                nn.Softmax(dim = 1))\n\n    \n    def forward(self, x):\n\n        x = self.model(x)\n        x = self.fc(x)\n        \n        return x\n    \nresnet_model = models.resnet18(pretrained=True)\nmodel = Neural(resnet_model, 10)\nmodel","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:00:26.054532Z","iopub.execute_input":"2021-10-18T09:00:26.054883Z","iopub.status.idle":"2021-10-18T09:00:32.702946Z","shell.execute_reply.started":"2021-10-18T09:00:26.054851Z","shell.execute_reply":"2021-10-18T09:00:32.702015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = optim.Adam(model.parameters(), lr=0.02)\nloss_func = nn.CrossEntropyLoss()\nscheduler = optim.lr_scheduler.OneCycleLR(opt, max_lr=0.003, steps_per_epoch=len(train_loader.dataset), epochs=10)\nmodel.to(device);","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:00:43.068297Z","iopub.execute_input":"2021-10-18T09:00:43.068715Z","iopub.status.idle":"2021-10-18T09:00:46.737292Z","shell.execute_reply.started":"2021-10-18T09:00:43.068677Z","shell.execute_reply":"2021-10-18T09:00:46.736387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\ndef train_model(models, loss, optimizer, train_data, val_data, scheduler=None, num_epoch=15):\n    \n    train_loss = []\n    train_acc = []\n    val_loss = []\n    val_acc = []\n    \n    best_train_acc, best_val_acc = 0, 0\n    \n    for epoch in tqdm(range(num_epoch)):\n        \n        models.train()\n        _train_loss, _train_acc = 0, 0\n            \n        for batch, (xb, yb) in enumerate(train_data):\n            xb, yb = xb.to(device), yb.to(device)\n            pred = models(xb)\n            yb = torch.squeeze(yb).long()\n\n            _loss = loss(pred, yb)\n            \n            _, pred = torch.max(pred, dim=1)\n            \n            _train_acc += torch.sum(pred==yb).item()\n            _train_loss += _loss.item()\n            _loss.backward()\n            \n            \n            optimizer.step()\n            optimizer.zero_grad()\n            if scheduler is not None:\n                scheduler.step()  \n            if batch % 200 == 0:\n                print(str(epoch) + \"  loss:  \" + str(_loss.item()))\n         \n        accuracy = 100 * _train_acc / len(train_data.dataset)\n        print('Best accuracy train epoch: ' + str(epoch) + ' ' + str(accuracy))\n        train_loss.append(_train_loss / len(train_data.dataset))\n        train_acc.append(accuracy)\n        models.eval()\n        with torch.no_grad():\n            _val_loss, _val_acc = 0, 0\n            for val_x, val_y in val_data:\n                val_x, val_y = val_x.to(device), val_y.to(device)\n                pred = models(val_x)\n                _val_loss += loss(pred, val_y).item()\n                _, preds = torch.max(pred, 1)\n                _val_acc += torch.sum(preds == val_y.data)\n                \n            val_accuracy = 100 * _val_acc.item() / len(val_data.dataset)\n            print(\"Best accuracy val epoch \" + str(epoch) + ' = ' + str(val_accuracy))\n            if val_accuracy >= best_val_acc:\n                print('Save best model')\n                torch.save(model.state_dict(), f\"test.pth\")\n                best_val_acc = val_accuracy\n            val_loss.append(_val_loss / len(val_data.dataset))\n            val_acc.append(val_accuracy)\n            \n         \n    return models, train_loss, train_acc, val_acc, val_loss\n\nnew_model, train_loss, train_acc, val_acc, val_loss = train_model(model, loss_func, opt, train_loader, val_loader, scheduler=scheduler, num_epoch=10)","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:00:48.598715Z","iopub.execute_input":"2021-10-18T09:00:48.599036Z","iopub.status.idle":"2021-10-18T09:02:47.809308Z","shell.execute_reply.started":"2021-10-18T09:00:48.599007Z","shell.execute_reply":"2021-10-18T09:02:47.807193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_loss_and_accuracy(train_loss, val_loss, train_acc, val_acc, title):\n\n    fig, (ax1, ax2) = plt.subplots(figsize=(20, 12), nrows=2)\n\n    x = list(range(len(train_loss)))\n\n    ax1.plot(x, train_loss,  label='train_loss')\n    ax1.plot(x, val_loss, label='val_loss')\n    ax1.set_title(\"loss in train and val\")\n    ax1.set_xlabel('Epoch')\n    ax1.set_ylabel('loss')\n    ax1.legend()\n\n    ax2.plot(x, train_acc, label='train_accuracy')\n    ax2.plot(x, val_acc, label='val_accuracy')\n    ax2.set_title(\"Accuracy in train and val\")\n    ax2.set_xlabel('Epoch')\n    ax2.set_ylabel('Accuracy')\n\n    fig.show((ax1, ax2))\n        \n        \nshow_loss_and_accuracy(train_loss, val_loss, train_acc, val_acc, title='loss')","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:02:58.973286Z","iopub.execute_input":"2021-10-18T09:02:58.973694Z","iopub.status.idle":"2021-10-18T09:02:59.260694Z","shell.execute_reply.started":"2021-10-18T09:02:58.973657Z","shell.execute_reply":"2021-10-18T09:02:59.259813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor n, param in enumerate(new_model.parameters()):\n    count += 1\n    if n >= 50:\n        param.requires_grad = True\n\ncount","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:03:08.852901Z","iopub.execute_input":"2021-10-18T09:03:08.853253Z","iopub.status.idle":"2021-10-18T09:03:08.861682Z","shell.execute_reply.started":"2021-10-18T09:03:08.85322Z","shell.execute_reply":"2021-10-18T09:03:08.860587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = optim.Adam(new_model.parameters(), lr=0.001)\nloss_func = nn.CrossEntropyLoss()\nscheduler = optim.lr_scheduler.OneCycleLR(opt, max_lr=0.0001, steps_per_epoch=len(train_loader.dataset), epochs=10)\nnew_model.to(device);","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:04:45.397457Z","iopub.execute_input":"2021-10-18T09:04:45.39783Z","iopub.status.idle":"2021-10-18T09:04:45.409414Z","shell.execute_reply.started":"2021-10-18T09:04:45.397798Z","shell.execute_reply":"2021-10-18T09:04:45.408598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_model, train_loss, train_acc, val_acc, val_loss = train_model(new_model, loss_func, opt, train_loader, val_loader, scheduler=scheduler, num_epoch=10)","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:04:49.316125Z","iopub.execute_input":"2021-10-18T09:04:49.316485Z","iopub.status.idle":"2021-10-18T09:06:48.349294Z","shell.execute_reply.started":"2021-10-18T09:04:49.316455Z","shell.execute_reply":"2021-10-18T09:06:48.346857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predicts(model, data):\n    model.eval()\n    test_preds = torch.LongTensor()\n    for i, item in enumerate(data):\n        item = item.to(device)\n        out = model(item)\n        _, preds = torch.max(out.data, 1)\n        test_preds = torch.cat((test_preds, preds.cpu()), dim = 0)\n        \n    return test_preds","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:07:13.702505Z","iopub.execute_input":"2021-10-18T09:07:13.702845Z","iopub.status.idle":"2021-10-18T09:07:13.71052Z","shell.execute_reply.started":"2021-10-18T09:07:13.702813Z","shell.execute_reply":"2021-10-18T09:07:13.709541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set_pred = predicts(new_model, test_loader)\n\nsubm_df = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\n\nsubm_df['Label'] = test_set_pred\nsubm_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:07:41.196239Z","iopub.execute_input":"2021-10-18T09:07:41.196603Z","iopub.status.idle":"2021-10-18T09:07:43.945876Z","shell.execute_reply.started":"2021-10-18T09:07:41.196571Z","shell.execute_reply":"2021-10-18T09:07:43.944987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm_df.to_csv('submit_test.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-10-18T09:18:21.175133Z","iopub.execute_input":"2021-10-18T09:18:21.17558Z","iopub.status.idle":"2021-10-18T09:18:21.230257Z","shell.execute_reply.started":"2021-10-18T09:18:21.175541Z","shell.execute_reply":"2021-10-18T09:18:21.229426Z"},"trusted":true},"execution_count":null,"outputs":[]}]}