{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing libraries and downloading datasets","metadata":{"id":"7ZDoqx5LqRK0"}},{"cell_type":"code","source":"import torch\nimport random\nimport numpy as np\nimport pickle\nimport os\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms\nimport matplotlib.pyplot as plt\n\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"id":"s0nMUZ48qWBW","outputId":"c01fb425-72ca-4bcd-d1ca-c2b17d55f1cd","execution":{"iopub.status.busy":"2022-05-25T18:23:06.330644Z","iopub.execute_input":"2022-05-25T18:23:06.331362Z","iopub.status.idle":"2022-05-25T18:23:06.339812Z","shell.execute_reply.started":"2022-05-25T18:23:06.331324Z","shell.execute_reply":"2022-05-25T18:23:06.338940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# уберем рандомность из вычислений\nrandom.seed(0)\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\ntorch.backends.cudnn.deterministic = True","metadata":{"id":"zu1wFArJqeNq","execution":{"iopub.status.busy":"2022-05-25T18:23:06.454085Z","iopub.execute_input":"2022-05-25T18:23:06.454946Z","iopub.status.idle":"2022-05-25T18:23:06.460315Z","shell.execute_reply.started":"2022-05-25T18:23:06.454893Z","shell.execute_reply":"2022-05-25T18:23:06.459613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get train data\ntrain = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:23:06.541275Z","iopub.execute_input":"2022-05-25T18:23:06.541527Z","iopub.status.idle":"2022-05-25T18:23:08.947500Z","shell.execute_reply.started":"2022-05-25T18:23:06.541496Z","shell.execute_reply":"2022-05-25T18:23:08.946798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# split dataframe to dataframe with data and labels\ny  = train[[\"label\"]]\nX = train.drop(\"label\", axis=1)\nX.head()","metadata":{"id":"hq_a2qwJqROv","outputId":"5a0adae3-4a04-4cda-b306-719f27ca89c5","execution":{"iopub.status.busy":"2022-05-25T18:23:08.949063Z","iopub.execute_input":"2022-05-25T18:23:08.949757Z","iopub.status.idle":"2022-05-25T18:23:09.051360Z","shell.execute_reply.started":"2022-05-25T18:23:08.949704Z","shell.execute_reply":"2022-05-25T18:23:09.050576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.head()","metadata":{"id":"mxTgSetcqkxN","outputId":"62ba4434-8435-40e6-b9d9-361f7641bf64","execution":{"iopub.status.busy":"2022-05-25T18:23:09.055301Z","iopub.execute_input":"2022-05-25T18:23:09.058240Z","iopub.status.idle":"2022-05-25T18:23:09.067835Z","shell.execute_reply.started":"2022-05-25T18:23:09.058197Z","shell.execute_reply":"2022-05-25T18:23:09.067123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split data to train and test\nX_train_pd, X_test_pd, y_train_pd, y_test_pd = train_test_split(X, y, test_size=0.15)","metadata":{"id":"h-xU9vLEqyqR","execution":{"iopub.status.busy":"2022-05-25T18:23:09.070131Z","iopub.execute_input":"2022-05-25T18:23:09.070632Z","iopub.status.idle":"2022-05-25T18:23:09.323786Z","shell.execute_reply.started":"2022-05-25T18:23:09.070595Z","shell.execute_reply":"2022-05-25T18:23:09.323007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert data to needed format\nX_train= torch.from_numpy(X_train_pd.to_numpy()).type(torch.LongTensor)\nX_test= torch.from_numpy(X_test_pd.to_numpy()).type(torch.LongTensor)\n\ny_train= torch.from_numpy(y_train_pd.to_numpy()).type(torch.LongTensor).view(-1)\ny_test = torch.from_numpy(y_test_pd.to_numpy()).type(torch.LongTensor).view(-1)","metadata":{"id":"pB8gBxxJr9YR","execution":{"iopub.status.busy":"2022-05-25T18:23:09.325074Z","iopub.execute_input":"2022-05-25T18:23:09.325325Z","iopub.status.idle":"2022-05-25T18:23:09.332132Z","shell.execute_reply.started":"2022-05-25T18:23:09.325292Z","shell.execute_reply":"2022-05-25T18:23:09.331473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape, \"\\n\", X_test.shape, \"\\n\", y_train.shape, \"\\n\", y_test.shape, \"\\n\" )","metadata":{"id":"9KPpJxk8t0BZ","outputId":"8fc09586-4d14-4bf5-83a1-f9b38d2853b4","execution":{"iopub.status.busy":"2022-05-25T18:23:09.333464Z","iopub.execute_input":"2022-05-25T18:23:09.333985Z","iopub.status.idle":"2022-05-25T18:23:09.344083Z","shell.execute_reply.started":"2022-05-25T18:23:09.333944Z","shell.execute_reply":"2022-05-25T18:23:09.343172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# посмотрим на тип тренировочных данных\nX_train.dtype, y_train.dtype","metadata":{"id":"c0AxXajmqwN_","outputId":"af127adf-5504-42e9-c0fc-21f6b366bc50","execution":{"iopub.status.busy":"2022-05-25T18:23:09.346935Z","iopub.execute_input":"2022-05-25T18:23:09.347203Z","iopub.status.idle":"2022-05-25T18:23:09.353679Z","shell.execute_reply.started":"2022-05-25T18:23:09.347168Z","shell.execute_reply":"2022-05-25T18:23:09.352914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# преобразуем  в дробный формат для вычислений производной тензора\nX_train_copy = X_train.float()\nX_test_copy = X_test.float()","metadata":{"id":"qLQ3ysYKqxwj","execution":{"iopub.status.busy":"2022-05-25T18:23:09.355058Z","iopub.execute_input":"2022-05-25T18:23:09.355634Z","iopub.status.idle":"2022-05-25T18:23:09.456180Z","shell.execute_reply.started":"2022-05-25T18:23:09.355593Z","shell.execute_reply":"2022-05-25T18:23:09.455432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vizualize data\nplt.imshow(X_train_copy.view(-1, 28, 28)[0, :, :])\nplt.show()\nprint(y_train[0])","metadata":{"id":"SsHwIZu-swds","outputId":"29b317d6-4de4-43ab-d599-83d3afbd6168","execution":{"iopub.status.busy":"2022-05-25T18:23:09.457719Z","iopub.execute_input":"2022-05-25T18:23:09.458120Z","iopub.status.idle":"2022-05-25T18:23:09.626023Z","shell.execute_reply.started":"2022-05-25T18:23:09.458082Z","shell.execute_reply":"2022-05-25T18:23:09.625301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a model","metadata":{}},{"cell_type":"markdown","source":"## Creating fully-connected net","metadata":{"id":"v7ToMG8Mwemg"}},{"cell_type":"code","source":"class MNISTNet(torch.nn.Module):\n    def __init__(self, n_hidden_neurons):\n        \n        # inherit from a parent class\n        super(MNISTNet, self).__init__()\n        \"\"\" \n        create the firts level. We have 28 * 28 neurons as input because\n        it's the size of a pic\n        \"\"\" \n        self.fc1 = torch.nn.Linear(28 * 28, n_hidden_neurons)\n        # create an activation function\n        self.ac1 = torch.nn.Sigmoid()\n        # we have 10 classes, so we have 10 neurons as the output\n        self.fc2 = torch.nn.Linear(n_hidden_neurons, 10) \n    \n    # In forward function we describe the sequence of actions\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.ac1(x)\n        x = self.fc2(x)\n        return x\n    \nmnist_net = MNISTNet(100)","metadata":{"id":"4YaISbGMwgKG","execution":{"iopub.status.busy":"2022-05-25T18:23:09.629143Z","iopub.execute_input":"2022-05-25T18:23:09.629348Z","iopub.status.idle":"2022-05-25T18:23:09.636820Z","shell.execute_reply.started":"2022-05-25T18:23:09.629322Z","shell.execute_reply":"2022-05-25T18:23:09.635900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if GPU is available\ntorch.cuda.is_available()","metadata":{"id":"8IVY_O-q6NfG","outputId":"b7305534-a458-4037-c0dc-e1f1219560d0","execution":{"iopub.status.busy":"2022-05-25T18:23:09.638468Z","iopub.execute_input":"2022-05-25T18:23:09.638752Z","iopub.status.idle":"2022-05-25T18:23:09.651155Z","shell.execute_reply.started":"2022-05-25T18:23:09.638698Z","shell.execute_reply":"2022-05-25T18:23:09.650232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the information about GPU\n!nvidia-smi","metadata":{"id":"d2dpB-HE6PL8","outputId":"1cb4e853-30a3-477d-d147-8c13e5abc9e5","execution":{"iopub.status.busy":"2022-05-25T18:23:09.652480Z","iopub.execute_input":"2022-05-25T18:23:09.652924Z","iopub.status.idle":"2022-05-25T18:23:10.406295Z","shell.execute_reply.started":"2022-05-25T18:23:09.652888Z","shell.execute_reply":"2022-05-25T18:23:10.405261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# turn on GPU for calculation\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nmnist_net = mnist_net.to(device)\n\n# we can check the initial parameter of our net\nlist(mnist_net.parameters())","metadata":{"id":"lQhQv6gX6We9","outputId":"a6320c68-dd08-48bf-d088-27a8794a43d0","execution":{"iopub.status.busy":"2022-05-25T18:23:10.408044Z","iopub.execute_input":"2022-05-25T18:23:10.409032Z","iopub.status.idle":"2022-05-25T18:23:10.449141Z","shell.execute_reply.started":"2022-05-25T18:23:10.408992Z","shell.execute_reply":"2022-05-25T18:23:10.447151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set the loss-funct and optimizer for the net\nloss = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(mnist_net.parameters(), lr=1.0e-3)","metadata":{"id":"fF1EApRkwikI","execution":{"iopub.status.busy":"2022-05-25T18:23:10.450538Z","iopub.execute_input":"2022-05-25T18:23:10.451248Z","iopub.status.idle":"2022-05-25T18:23:10.457522Z","shell.execute_reply.started":"2022-05-25T18:23:10.451209Z","shell.execute_reply":"2022-05-25T18:23:10.456571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbatch_size = 100\n\n# they are needed for vizualizing loss-func and accuracy plot\ntest_accuracy_history = []\ntest_loss_history = []\n\n# turn on GPU for calculation\nX_test_copy = X_test_copy.to(device)\ny_test = y_test.to(device)\n\nfor epoch in range(250):\n    # shuffle our data\n    order = np.random.permutation(len(X_train_copy))\n    \n    for start_index in range(0, len(X_train_copy), batch_size):\n        \n        # we need to make our gradient null after each iteration. Otherwise they'll be summing.\n        optimizer.zero_grad()\n        \n        # get needed indexes for a batch\n        batch_indexes = order[start_index:start_index+batch_size]\n        \n        # leave the data with needed indexes\n        X_batch = X_train_copy[batch_indexes].to(device)\n        y_batch = y_train[batch_indexes].to(device)\n        \n        preds = mnist_net.forward(X_batch) \n        \n        loss_value = loss(preds, y_batch)\n        \n        # get the derivitive(gradient)\n        loss_value.backward()\n        \n        # making gradient step - updating our parameters\n        optimizer.step()\n\n    test_preds = mnist_net.forward(X_test_copy)\n    test_loss_history.append(loss(test_preds, y_test).cpu().detach().numpy())\n    \n    accuracy = (test_preds.argmax(dim=1) == y_test).float().mean().cpu().detach().numpy()\n    test_accuracy_history.append(accuracy)\n    if epoch % 100 == 0:\n        print(\"epoch =\", epoch,accuracy)","metadata":{"id":"cSx02mw2xJbo","outputId":"7083795b-d893-4115-bbbe-fa63efc4c6ae","execution":{"iopub.status.busy":"2022-05-25T18:23:10.459046Z","iopub.execute_input":"2022-05-25T18:23:10.460051Z","iopub.status.idle":"2022-05-25T18:26:54.358147Z","shell.execute_reply.started":"2022-05-25T18:23:10.459894Z","shell.execute_reply":"2022-05-25T18:26:54.357355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # save the results in order not to train model every time\n# try:\n#     with open('test_accuracy_history.pickle', 'rb') as f:\n#         test_accuracy_history = pickle.load(f)\n#     with open('test_loss_history.pickle', 'rb') as f:\n#         test_loss_history = pickle.load(f)\n    \n# except:\n#     with open(\"test_accuracy_history.pickle\", 'wb') as f:\n#         pickle.dump(test_accuracy_history, f)\n#     with open(\"test_loss_history.pickle\", 'wb') as f:\n#         pickle.dump(test_loss_history, f)\n","metadata":{"id":"N3Is-N6JBrLr","execution":{"iopub.status.busy":"2022-05-25T18:26:54.359494Z","iopub.execute_input":"2022-05-25T18:26:54.360328Z","iopub.status.idle":"2022-05-25T18:26:54.364873Z","shell.execute_reply.started":"2022-05-25T18:26:54.360289Z","shell.execute_reply":"2022-05-25T18:26:54.364084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\nplt.title(\"Зависимость accuracy от количества эпох\", fontsize=20)\nplt.xlabel('epoch', fontsize=16)\nplt.ylabel('accuracy', fontsize=16)\nplt.plot(test_accuracy_history)\n","metadata":{"id":"6zEd3HN6oeky","outputId":"98fadcc4-49e9-44ce-80fa-25cd1a01a27f","execution":{"iopub.status.busy":"2022-05-25T18:26:54.366210Z","iopub.execute_input":"2022-05-25T18:26:54.366461Z","iopub.status.idle":"2022-05-25T18:26:54.578506Z","shell.execute_reply.started":"2022-05-25T18:26:54.366424Z","shell.execute_reply":"2022-05-25T18:26:54.577817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\nplt.title(\"Зависимость значений loss-function от количества эпох\", fontsize=20)\nplt.xlabel('epoch', fontsize=16)\nplt.ylabel('value of loss-function', fontsize=16)\nplt.plot(test_loss_history)","metadata":{"id":"TCi8DZhNXE3p","outputId":"9ab25e25-c3f6-4a6b-e861-b5aff415c91d","execution":{"iopub.status.busy":"2022-05-25T18:26:54.579800Z","iopub.execute_input":"2022-05-25T18:26:54.580154Z","iopub.status.idle":"2022-05-25T18:26:54.775498Z","shell.execute_reply.started":"2022-05-25T18:26:54.580114Z","shell.execute_reply":"2022-05-25T18:26:54.774811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get the maximum and minimum value of the metric","metadata":{"id":"BKmQxf3FgOpz"}},{"cell_type":"code","source":"max(test_accuracy_history)","metadata":{"id":"6VnG1SGM7Mwd","outputId":"01b13aee-cfc5-47a2-933e-27a361d01ea9","execution":{"iopub.status.busy":"2022-05-25T18:26:54.776769Z","iopub.execute_input":"2022-05-25T18:26:54.777136Z","iopub.status.idle":"2022-05-25T18:26:54.785119Z","shell.execute_reply.started":"2022-05-25T18:26:54.777099Z","shell.execute_reply":"2022-05-25T18:26:54.784179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(test_loss_history)","metadata":{"id":"y-rPoh-bfwND","outputId":"9f660800-5e1a-4f21-b973-cae5e283386b","execution":{"iopub.status.busy":"2022-05-25T18:26:54.787070Z","iopub.execute_input":"2022-05-25T18:26:54.787482Z","iopub.status.idle":"2022-05-25T18:26:54.795957Z","shell.execute_reply.started":"2022-05-25T18:26:54.787381Z","shell.execute_reply":"2022-05-25T18:26:54.795193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating convolutional net","metadata":{"id":"Gaq-8O6Q6TGw"}},{"cell_type":"markdown","source":"Let's see if another type of neural net performs the task better\n","metadata":{"id":"0g-GtbVpdqQT"}},{"cell_type":"code","source":"# transform datasets to send them to the net\nX_train_conv = X_train.view(-1, 1, 28, 28).float()\nX_test_conv = X_test.view(-1, 1, 28, 28).float()","metadata":{"id":"yPGB0030-wEO","execution":{"iopub.status.busy":"2022-05-25T18:26:54.797189Z","iopub.execute_input":"2022-05-25T18:26:54.797587Z","iopub.status.idle":"2022-05-25T18:26:54.906029Z","shell.execute_reply.started":"2022-05-25T18:26:54.797550Z","shell.execute_reply":"2022-05-25T18:26:54.905141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_conv.shape","metadata":{"id":"waqVG04y-wOV","outputId":"41468b11-9167-4a0c-a79d-359857fa3e77","execution":{"iopub.status.busy":"2022-05-25T18:26:54.907456Z","iopub.execute_input":"2022-05-25T18:26:54.907766Z","iopub.status.idle":"2022-05-25T18:26:54.914107Z","shell.execute_reply.started":"2022-05-25T18:26:54.907707Z","shell.execute_reply":"2022-05-25T18:26:54.913305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n- batch-normalization should make the process of fitting a model faster;\n- max-pooling should gives better results, in practice, than average-pooling;\n- Tanh() as activation function leads to gradient decay. It means that in process of backpropagation the wieghts of\nfirst layers will be not able to update, thus we have to use other act-func to avoid this.\n- 2 convolutional layers with 3x3 kernel gives the same result as 1 layer with 5x5 kernel. Using 2 layers\n3x3 we have to update 3x3x2=18 parameters but when using 1 layer with 5x5 we have to update 5x5=25 parameters.\nFor more information read about receptive field.\n\"\"\"\nclass LeNet5(torch.nn.Module):\n    def __init__(self,\n                 activation='tanh',\n                 pooling='avg',\n                 conv_size=5,\n                 use_batch_norm=False):\n        super(LeNet5, self).__init__()\n        \n        self.conv_size = conv_size\n        self.use_batch_norm = use_batch_norm\n        \n        if activation == 'tanh':\n            activation_function = torch.nn.Tanh()\n        elif activation == 'relu':\n            activation_function  = torch.nn.ReLU()\n        else:\n            raise NotImplementedError\n            \n        if pooling == 'avg':\n            pooling_layer = torch.nn.AvgPool2d(kernel_size=2, stride=2)\n        elif pooling == 'max':\n            pooling_layer  = torch.nn.MaxPool2d(kernel_size=2, stride=2)\n        else:\n            raise NotImplementedError\n        # first conv layer\n        if conv_size == 5:\n            self.conv1 = torch.nn.Conv2d(\n                in_channels=1, out_channels=16, kernel_size=5, padding=2)\n        elif conv_size == 3:\n            self.conv1_1 = torch.nn.Conv2d(\n                in_channels=1, out_channels=16, kernel_size=3, padding=1)\n            self.conv1_2 = torch.nn.Conv2d(\n                in_channels=16, out_channels=16, kernel_size=3, padding=1)\n        else:\n            raise NotImplementedError\n\n        self.act1 = activation_function\n        self.bn1 = torch.nn.BatchNorm2d(num_features=16)\n        self.pool1 = pooling_layer\n        # dropout helps us to avoid overfitting\n        self.dropout1 = torch.nn.Dropout(0.1)\n        # first conv layer\n       \n        # second conv layer\n        if conv_size == 5:\n            self.conv2 = torch.nn.Conv2d(\n                in_channels=16, out_channels=32, kernel_size=5, padding=4)\n        elif conv_size == 3:\n            self.conv2_1 = torch.nn.Conv2d(\n                in_channels=16, out_channels=32, kernel_size=3, padding=0)\n            self.conv2_2 = torch.nn.Conv2d(\n                in_channels=32, out_channels=32, kernel_size=3, padding=0)\n        else:\n            raise NotImplementedError\n\n        self.act2 = activation_function\n        self.bn2 = torch.nn.BatchNorm2d(num_features=32)\n        self.pool2 = pooling_layer\n        self.dropout2 = torch.nn.Dropout(0.2)\n        # second conv layer\n\n        #third conv layer\n        if conv_size == 5:\n            self.conv3 = torch.nn.Conv2d(\n                in_channels=32, out_channels=64, kernel_size=5, padding=0)\n        elif conv_size == 3:\n            self.conv3_1 = torch.nn.Conv2d(\n                in_channels=32, out_channels=64, kernel_size=3, padding=1)\n            self.conv3_2 = torch.nn.Conv2d(\n                in_channels=64, out_channels=64, kernel_size=3, padding=1)\n        else:\n            raise NotImplementedError\n\n        self.act3 = activation_function\n        self.bn3 = torch.nn.BatchNorm2d(num_features=64)\n        self.dropout3 = torch.nn.Dropout(0.3)        \n        #third conv layer\n\n        # first fully-connected layer\n        self.fc1 = torch.nn.Linear(5 * 5 * 64, 120)\n        self.act4 = activation_function\n        \n        # second fully-connected layer\n        self.fc2 = torch.nn.Linear(120, 84)\n        self.act5 = activation_function\n        \n        # third fully-connected layer\n        self.fc3 = torch.nn.Linear(84, 10)\n        \n        \"\"\"\n        We don't use the softmax necessarily because we can calculate loss-function and metric without it.\n        It's because there are logorithm and exponent in the formulas of CrossEntropy and Softmax.\n        So they can cut each other out. For more information see the folmulas)))\n        \"\"\"\n    \n    def forward(self, x):\n        if self.conv_size == 5:\n            x = self.conv1(x)\n        elif self.conv_size == 3:\n            x = self.conv1_2(self.conv1_1(x))\n        x = self.act1(x)\n        if self.use_batch_norm:\n            x = self.bn1(x)\n        x = self.pool1(x)\n        x = self.dropout1(x)\n        # print(x.shape)\n        \n        \n        if self.conv_size == 5:\n            x = self.conv2(x)\n        elif self.conv_size == 3:\n            x = self.conv2_2(self.conv2_1(x))\n        x = self.act2(x)\n        if self.use_batch_norm:\n            x = self.bn2(x)\n        x = self.pool2(x)\n        x = self.dropout2(x)\n        # print(x.shape)\n\n        \n        if self.conv_size == 5:\n            x = self.conv3(x)\n        elif self.conv_size == 3:\n            x = self.conv3_2(self.conv3_1(x))\n        x = self.act3(x)\n        if self.use_batch_norm:\n            x = self.bn3(x)\n        x = self.dropout3(x)\n        # print(x.shape)        \n\n\n        x = x.view(x.size(0), x.size(1) * x.size(2) * x.size(3))\n        x = self.fc1(x)\n        x = self.act4(x)\n        x = self.fc2(x)\n        x = self.act5(x)\n        x = self.fc3(x)\n        \n        return x","metadata":{"id":"hcyo7n7i6XFK","execution":{"iopub.status.busy":"2022-05-25T18:26:54.915856Z","iopub.execute_input":"2022-05-25T18:26:54.916349Z","iopub.status.idle":"2022-05-25T18:26:54.948668Z","shell.execute_reply.started":"2022-05-25T18:26:54.916306Z","shell.execute_reply":"2022-05-25T18:26:54.947679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\"\nTo use batch-normalization correctly we have to use net.train() and net.eval(). While using batch-norm we calculate\nupdate values of standard deviation and mean, thus we have to exclude updating those values in testing a net.\nSo we have to set it explicitly when we train and when we test\n\"\"\"\n\ndef train(net, X_train, y_train, X_test, y_test):\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    net = net.to(device)\n    loss = torch.nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(net.parameters(), lr=1.0e-4)\n    \n    batch_size = 100\n\n    test_accuracy_history = []\n    test_loss_history = []\n\n    X_test = X_test.to(device)\n    y_test = y_test.to(device)\n\n    for epoch in range(101):\n        order = np.random.permutation(len(X_train))\n        for start_index in range(0, len(X_train), batch_size):\n            optimizer.zero_grad()\n            net.train()\n\n            batch_indexes = order[start_index:start_index+batch_size]\n\n            X_batch = X_train[batch_indexes].to(device)\n            y_batch = y_train[batch_indexes].to(device)\n\n            preds = net.forward(X_batch) \n\n            loss_value = loss(preds, y_batch)\n            loss_value.backward()\n\n            optimizer.step()\n\n        net.eval()\n        test_preds = net.forward(X_test)\n        test_loss_history.append(loss(test_preds, y_test).data.cpu())\n\n        accuracy = (test_preds.argmax(dim=1) == y_test).float().mean().data.cpu()\n        test_accuracy_history.append(accuracy)\n\n        if epoch % 10 == 0:\n            print(\"epoch =\", epoch,accuracy)        \n    print('---------------')\n    return test_accuracy_history, test_loss_history\n\naccuracies = {}\nlosses = {}\n\n\"\"\"\nLet's explore which model does better\n\"\"\"\naccuracies['tanh'], losses['tanh'] = \\\n    train(LeNet5(activation='tanh', conv_size=5),\n          X_train_conv, y_train, X_test_conv, y_test)\n\naccuracies['relu'], losses['relu'] = \\\n    train(LeNet5(activation='relu', conv_size=5),\n          X_train_conv, y_train, X_test_conv, y_test)\n\naccuracies['relu_3'], losses['relu_3'] = \\\n    train(LeNet5(activation='relu', conv_size=3),\n          X_train_conv, y_train, X_test_conv, y_test)\n\naccuracies['relu_3_max_pool'], losses['relu_3_max_pool'] = \\\n    train(LeNet5(activation='relu', conv_size=3, pooling='max'), \n          X_train_conv, y_train, X_test_conv, y_test)\n\naccuracies['relu_3_max_pool_bn'], losses['relu_3_max_pool_bn'] = \\\n    train(LeNet5(activation='relu', conv_size=3, pooling='max', use_batch_norm=True), \n          X_train_conv, y_train, X_test_conv, y_test)","metadata":{"id":"I89m6tf3IvOA","outputId":"01d2340d-a61f-4679-ba4d-44e264fff37a","execution":{"iopub.status.busy":"2022-05-25T18:26:54.952589Z","iopub.execute_input":"2022-05-25T18:26:54.952839Z","iopub.status.idle":"2022-05-25T18:45:28.398790Z","shell.execute_reply.started":"2022-05-25T18:26:54.952802Z","shell.execute_reply":"2022-05-25T18:45:28.398014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # save results in order not to train model every time\n# try:\n#     with open('accuracies.pickle', 'rb') as f:\n#         accuracies = pickle.load(f)\n#     with open('losses.pickle', 'rb') as f:\n#     losses = pickle.load(f)\n\n# except:\n#     with open(\"losses.pickle\", 'wb') as f:\n#         pickle.dump(losses, f)\n#     with open(\"accuracies.pickle\", 'wb') as f:\n#     pickle.dump(accuracies, f)\n","metadata":{"id":"DYD1to-kBZSu","execution":{"iopub.status.busy":"2022-05-25T18:45:28.400363Z","iopub.execute_input":"2022-05-25T18:45:28.400831Z","iopub.status.idle":"2022-05-25T18:45:28.405272Z","shell.execute_reply.started":"2022-05-25T18:45:28.400786Z","shell.execute_reply":"2022-05-25T18:45:28.404527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nfor experiment_id in accuracies.keys():\n    plt.plot(accuracies[experiment_id], label=experiment_id)\nplt.legend()\nplt.title('Validation Accuracy');","metadata":{"id":"vHAUPlU0ua2h","execution":{"iopub.status.busy":"2022-05-25T18:45:28.406615Z","iopub.execute_input":"2022-05-25T18:45:28.407072Z","iopub.status.idle":"2022-05-25T18:45:28.653472Z","shell.execute_reply.started":"2022-05-25T18:45:28.407030Z","shell.execute_reply":"2022-05-25T18:45:28.652793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nfor experiment_id in losses.keys():\n    plt.plot(losses[experiment_id], label=experiment_id)\nplt.legend()\nplt.title('Validation Loss');","metadata":{"id":"J4JgnqMSuePs","execution":{"iopub.status.busy":"2022-05-25T18:45:28.654778Z","iopub.execute_input":"2022-05-25T18:45:28.655154Z","iopub.status.idle":"2022-05-25T18:45:28.876234Z","shell.execute_reply.started":"2022-05-25T18:45:28.655115Z","shell.execute_reply":"2022-05-25T18:45:28.875522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_value_dict = dict()\nfor key in accuracies.keys():\n    max_value = max(accuracies.get(key))\n    max_value_dict[key] = max_value\nmax_value","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:45:28.880556Z","iopub.execute_input":"2022-05-25T18:45:28.880990Z","iopub.status.idle":"2022-05-25T18:45:28.890120Z","shell.execute_reply.started":"2022-05-25T18:45:28.880945Z","shell.execute_reply":"2022-05-25T18:45:28.889249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Вывод","metadata":{"id":"ZB5u6btzEw88"}},{"cell_type":"markdown","source":"1) ***Comparing convolutional nets and fully-connected net.***\n\nAs we can see that convolutional nets perform better. The max value of fully-connected net is about 0.96 and of conv net is more than 0.99\n\n2) ***Comparing of conv nets.***\n\n**Looking at accuracy plot** we can see that the model with norm-batch, ReLu as act-func and 3x3 kernels with max-pooling performs better than others. Also we have proven that bath-normalization helps us to speed up the process of fitting as its graph got the high values faster than others.\n\n**Looking at losses** plot we see that model with batch-norm has the least value of loss-func.\n\n**Choose the model with ReLu, batch-norm and 3x3 convolutional layers**","metadata":{"id":"l_Om_Pk_FBjj"}},{"cell_type":"markdown","source":"# Making predictions by the best model","metadata":{}},{"cell_type":"code","source":"# for training the best net\ndef fit(net, X_train, y_train, num_epoch=150):\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    net = net.to(device)\n    loss = torch.nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(net.parameters(), lr=1.0e-3)\n    \n    batch_size = 100\n\n    for epoch in range(num_epoch):\n        order = np.random.permutation(len(X_train))\n        for start_index in range(0, len(X_train), batch_size):\n            optimizer.zero_grad()\n            net.train()\n\n            batch_indexes = order[start_index:start_index+batch_size]\n\n            X_batch = X_train[batch_indexes].to(device)\n            y_batch = y_train[batch_indexes].to(device)\n\n            preds = net.forward(X_batch) \n\n            loss_value = loss(preds, y_batch)\n            loss_value.backward()\n\n            optimizer.step()\n\n        if epoch % 20 == 0:\n            print(f\"num_epoch = {epoch}\")\n","metadata":{"id":"Hg3inCtpBNR_","execution":{"iopub.status.busy":"2022-05-25T18:45:28.891555Z","iopub.execute_input":"2022-05-25T18:45:28.891825Z","iopub.status.idle":"2022-05-25T18:45:28.902118Z","shell.execute_reply.started":"2022-05-25T18:45:28.891788Z","shell.execute_reply":"2022-05-25T18:45:28.901292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = y_train.to(device)\n# Unite all data\nX_united = torch.cat([X_train_conv, X_test_conv])\ny_united = torch.cat([y_train, y_test])\n\nprint(X_united.shape, \"\\n\", y_united.shape)","metadata":{"id":"EDmX5UX0t6MF","execution":{"iopub.status.busy":"2022-05-25T18:47:35.537321Z","iopub.execute_input":"2022-05-25T18:47:35.537752Z","iopub.status.idle":"2022-05-25T18:47:35.726597Z","shell.execute_reply.started":"2022-05-25T18:47:35.537684Z","shell.execute_reply":"2022-05-25T18:47:35.725788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creating best model\nbest_net = LeNet5(activation='relu', conv_size=3, pooling='max', use_batch_norm=True)\n\n# # train the model\nfit(best_net, X_united, y_united, num_epoch=200)","metadata":{"id":"IhDHjE6J3nLN","execution":{"iopub.status.busy":"2022-05-25T18:47:41.216051Z","iopub.execute_input":"2022-05-25T18:47:41.216314Z","iopub.status.idle":"2022-05-25T18:57:20.851520Z","shell.execute_reply.started":"2022-05-25T18:47:41.216284Z","shell.execute_reply":"2022-05-25T18:57:20.850770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open(\"best_model.pickle\", 'wb') as f:\n#     pickle.dump(best_net, f)\n# with open('best_model.pickle', 'rb') as f:\n#     best_net = pickle.load(f)","metadata":{"id":"ASrbKmLD4mqC","execution":{"iopub.status.busy":"2022-05-25T18:57:20.853150Z","iopub.execute_input":"2022-05-25T18:57:20.853396Z","iopub.status.idle":"2022-05-25T18:57:20.858544Z","shell.execute_reply.started":"2022-05-25T18:57:20.853362Z","shell.execute_reply":"2022-05-25T18:57:20.857667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# download test data\ntest_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\n# convert pd.DataFrame to torch.tensor and add some dimensions in order to transfer it to net\ntorch_test = torch.from_numpy(test_data.to_numpy()).type(torch.LongTensor).view(-1, 1, 28, 28).float()","metadata":{"id":"Fg6FYlck7R2T","execution":{"iopub.status.busy":"2022-05-25T18:57:20.859842Z","iopub.execute_input":"2022-05-25T18:57:20.860652Z","iopub.status.idle":"2022-05-25T18:57:22.708541Z","shell.execute_reply.started":"2022-05-25T18:57:20.860613Z","shell.execute_reply":"2022-05-25T18:57:22.707785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get predictions. argmax(dim=1) is needed to get the most likely digit among all digits for a single picture\npreds = best_net.forward(torch_test.to(device)).argmax(dim=1)\npreds","metadata":{"id":"jG_GsGeZac1V","execution":{"iopub.status.busy":"2022-05-25T18:57:22.710476Z","iopub.execute_input":"2022-05-25T18:57:22.710759Z","iopub.status.idle":"2022-05-25T18:57:22.872335Z","shell.execute_reply.started":"2022-05-25T18:57:22.710710Z","shell.execute_reply":"2022-05-25T18:57:22.871513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert tensor to np.array\npred_np = preds.cpu().detach().numpy()\n\n# create empty DataFrame\nsubmission_df = pd.DataFrame()\n\n# create list of ImageId which shoud be sutable for Kagle\nimage_id = [i for i in range(1, 28001)]\n\nsubmission_df['ImageId'] = image_id\nsubmission_df['Label'] = pred_np\n\nsubmission_df","metadata":{"id":"WKg9U4NreDNt","execution":{"iopub.status.busy":"2022-05-25T18:57:22.873855Z","iopub.execute_input":"2022-05-25T18:57:22.874128Z","iopub.status.idle":"2022-05-25T18:57:22.915097Z","shell.execute_reply.started":"2022-05-25T18:57:22.874093Z","shell.execute_reply":"2022-05-25T18:57:22.914265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index = False)","metadata":{"id":"Az3jAbwmgErx","execution":{"iopub.status.busy":"2022-05-25T18:57:22.916451Z","iopub.execute_input":"2022-05-25T18:57:22.916790Z","iopub.status.idle":"2022-05-25T18:57:22.961277Z","shell.execute_reply.started":"2022-05-25T18:57:22.916754Z","shell.execute_reply":"2022-05-25T18:57:22.960551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(\"submission.csv\").head()","metadata":{"id":"OxAYkrwrhijV","execution":{"iopub.status.busy":"2022-05-25T18:57:22.962523Z","iopub.execute_input":"2022-05-25T18:57:22.962786Z","iopub.status.idle":"2022-05-25T18:57:22.980001Z","shell.execute_reply.started":"2022-05-25T18:57:22.962752Z","shell.execute_reply":"2022-05-25T18:57:22.979302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ideas about improvement\nTo get better result we should choose the nets with other sturctures like AlexNet, VGG,GoogLeNet or ResNet. They are more advanced and allow us to have more layers without decaying of gradient.","metadata":{}}]}