{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-15T15:31:05.416773Z","iopub.execute_input":"2022-09-15T15:31:05.417216Z","iopub.status.idle":"2022-09-15T15:31:05.447973Z","shell.execute_reply.started":"2022-09-15T15:31:05.417075Z","shell.execute_reply":"2022-09-15T15:31:05.447010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The Beginning","metadata":{}},{"cell_type":"markdown","source":"Import required libs","metadata":{}},{"cell_type":"code","source":"import itertools\nimport tqdm\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch\n\nfrom torch.optim import lr_scheduler, Adam, SGD, RMSprop\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom numpy import genfromtxt","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:31:05.449612Z","iopub.execute_input":"2022-09-15T15:31:05.449873Z","iopub.status.idle":"2022-09-15T15:31:07.841693Z","shell.execute_reply.started":"2022-09-15T15:31:05.449842Z","shell.execute_reply":"2022-09-15T15:31:07.840663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load the dataset for training. Normalize pixel values from [0, 255] to [0., 1.]","metadata":{}},{"cell_type":"code","source":"mnist_data = np.genfromtxt('/kaggle/input/digit-recognizer/train.csv', delimiter=',',skip_header=1)\ntrain_set = mnist_data[:,1:].astype('float32') / 255.0\ntrain_labels = mnist_data[:,0].astype(np.compat.long)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:31:07.843460Z","iopub.execute_input":"2022-09-15T15:31:07.843824Z","iopub.status.idle":"2022-09-15T15:31:41.376015Z","shell.execute_reply.started":"2022-09-15T15:31:07.843790Z","shell.execute_reply":"2022-09-15T15:31:41.375032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Several examples from the dataset:","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(16,8))\nfor i in range(10):\n    fig.add_subplot(2, 5, 1+i)\n    plt.imshow(train_set[123 + i*1000].reshape(28,28))\n    plt.xlabel(train_labels[123 + i*1000])","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:31:41.377351Z","iopub.execute_input":"2022-09-15T15:31:41.377628Z","iopub.status.idle":"2022-09-15T15:31:42.575754Z","shell.execute_reply.started":"2022-09-15T15:31:41.377597Z","shell.execute_reply":"2022-09-15T15:31:42.574880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a class to simplify data access\nclass MyMnistDataset(Dataset):\n    def __init__(self, train_set, labels):\n        self.x = torch.from_numpy(train_set)\n        self.y = torch.from_numpy(labels)\n\n    def __len__(self):\n        return len(self.x)\n\n    def __getitem__(self, i):\n        return self.x[i], self.y[i]\n\n# the function splits a dataset into training and validation sets\ndef split_dataset(train_set, labels, amount=0.8):\n    return train_test_split(train_set, labels, test_size=1.0-amount, shuffle=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:31:42.577494Z","iopub.execute_input":"2022-09-15T15:31:42.577731Z","iopub.status.idle":"2022-09-15T15:31:42.584798Z","shell.execute_reply.started":"2022-09-15T15:31:42.577704Z","shell.execute_reply":"2022-09-15T15:31:42.583632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history):\n    plt.figure(figsize=(18, 18))\n    plt.subplot(211)\n    plt.title(\"Loss\")\n    plt.plot(history['train_loss'], color=\"blue\", label=\"Loss\")\n    plt.plot(history['val_loss'], color=\"red\", label=\"Val loss\")\n    plt.legend()\n\n    plt.subplot(212)\n    plt.title(\"Accuracy\")\n    plt.plot(history['train_acc'], color=\"blue\", label=\"Accuracy\")\n    plt.plot(history['val_acc'], color=\"red\", label=\"Val accuracy\")\n    plt.xlabel(\"Epoch\")\n    plt.legend()\n\n# the function either trains or validates the NN model and calculates the accuracy of the results\ndef train_or_validate(m, loader, to_train = True):\n    m.train(to_train)\n\n    loss_sum = correct = 0\n    for i, (data, mark) in enumerate(loader):\n        if torch.cuda.is_available():\n            data = data.cuda()\n            mark = mark.cuda()\n\n        output = m(data)\n        loss = m.loss_function(output, mark)\n        loss_sum += loss.item()\n        \n        prediction = output.max(1, keepdim=True)[1]\n        correct += (prediction.cpu() == mark.view_as(prediction).cpu()).sum()\n        \n        if to_train:\n            m.optimizer.zero_grad()\n            loss.backward()\n            m.optimizer.step()\n\n    average_loss = loss_sum / len(loader.dataset)\n    accuracy = 100.0 * correct / len(loader.dataset)\n    \n    return average_loss, accuracy\n\n# here we randomly split the dataset into training and validation sets, \n# run the model on the former, test it with the latter, and record the training history (accuracy and loss params) \ndef run(model):\n    history = {'val_loss': [], 'val_acc': [],  'train_loss': [],  'train_acc': [] }\n    train_x, val_x, train_y, val_y = split_dataset(train_set, train_labels)\n    train_dataset, val_dataset = MyMnistDataset(train_x, train_y), MyMnistDataset(val_x, val_y)\n        \n    for epoch in range(epochs):\n        train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=model.batch_size, shuffle=True)\n        train_loss, train_acc = train_or_validate(model, train_loader, True)\n    \n        val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=model.batch_size)\n        val_loss, val_acc = train_or_validate(model, val_loader, False)\n\n        history['val_loss'].append(val_loss)\n        history['val_acc'].append(val_acc)\n        history['train_acc'].append(train_acc)\n        history['train_loss'].append(train_loss)\n\n        model.exp_lr_scheduler.step()\n    return history\n\n# find the best configuration of hyperparameters from several models\ndef show_best_config(histories):\n    best_history = histories[0]\n    for h in histories[1:]:\n        if np.max(h[1]['val_acc']) > np.max(best_history[1]['val_acc']):\n            best_history = h\n    best_model, history = best_history\n    print(\"Best model config: \", vars(best_model))\n    best_val_acc_i = np.argmax(history['val_acc']);\n    print(f'Best val_acc: {history[\"val_acc\"][best_val_acc_i].item()} on epoch {best_val_acc_i+1}')\n    plot_history(history)\n    return best_history\n\n# as for the NN model itself, here we use simple multilayer perceptron with two layers only\nclass MyMultiLayerPerceptron(torch.nn.Module):\n    def __init__(self, batch_size, epochs, step_size, gamma, optimizer_type, lr_mult):\n        super().__init__()\n\n        self.batch_size = batch_size\n        self.epochs = epochs\n        self.step_size = step_size\n        self.gamma = gamma\n        self.optimizer_type = optimizer_type\n        self.lr_mult = lr_mult\n        \n        self.loss_function = torch.nn.CrossEntropyLoss() # cross-entropy is a pretty standard loss function for our tasks\n        self.layers = torch.nn.Sequential(\n            torch.nn.Linear(28*28, 100), # 28 * 28 = 768 pixels for each image\n            torch.nn.ReLU(),\n            torch.nn.Linear(100, 10),\n        )\n\n        if (optimizer_type == \"SGD\"):\n            self.optimizer = SGD(super().parameters(), lr=0.01)\n        elif (optimizer_type == \"Nesterov\"):\n            self.optimizer = SGD(super().parameters(), lr=0.01, momentum=0.9, nesterov=True)\n        elif (optimizer_type == \"AMSGrad\"):\n            self.optimizer = Adam(super().parameters(), amsgrad=True)    \n        elif (optimizer_type == \"Adam\"):\n            self.optimizer = Adam(super().parameters())    \n        else:\n            self.optimizer = RMSprop(super().parameters())\n        self.optimizer.param_groups[0]['lr'] *= lr_mult\n\n        self.exp_lr_scheduler = lr_scheduler.StepLR(self.optimizer, step_size=self.step_size, gamma=self.gamma)\n\n    def forward(self, x):\n        x.view(x.size(0), -1)\n        return self.layers(x)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:31:42.586593Z","iopub.execute_input":"2022-09-15T15:31:42.587185Z","iopub.status.idle":"2022-09-15T15:31:42.613521Z","shell.execute_reply.started":"2022-09-15T15:31:42.587141Z","shell.execute_reply":"2022-09-15T15:31:42.612608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next we try several different combinations of hyperparameters (optimizer alg, learning rate, etc).\n\nEach obtained model is tested on the validation set. Our goal here is to find the best model to run it on the Kaggle's testing data","metadata":{}},{"cell_type":"code","source":"histories = []\nproducts = list(itertools.product(\n    ['SGD', 'Nesterov', 'AMSGrad', 'RMSprop', 'Adam'],  # optimizers\n    [1, 1/2, 2, 1/5, 5, 1/10],                          # lr_mults\n    [15, 30],                                           # epochs\n    [64, 128],                                          # batch_sizes\n    [1, 3],                                             # lr_scheduler step\n    [1.0, 0.3],                                         # lr_scheduler gamma\n))\n\n\nfor i, params in tqdm.notebook.tqdm(enumerate(products), total=len(products)):\n    optimizer, lr_mult, epochs, batch_size, step_size, gamma = params\n\n    model = MyMultiLayerPerceptron(batch_size, epochs, step_size, gamma, optimizer, lr_mult)\n    if torch.cuda.is_available():\n        model = model.cuda()\n        model.loss_function = model.loss_function.cuda()\n\n    history = run(model)\n    histories.append((model, history))\n\nbest_model, history = show_best_config(histories)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:31:42.614774Z","iopub.execute_input":"2022-09-15T15:31:42.615175Z","iopub.status.idle":"2022-09-15T15:32:20.179930Z","shell.execute_reply.started":"2022-09-15T15:31:42.615127Z","shell.execute_reply":"2022-09-15T15:32:20.178960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the best model on the test data","metadata":{}},{"cell_type":"code","source":"mnist_test_data = np.genfromtxt('/kaggle/input/digit-recognizer/test.csv', delimiter=',',skip_header=1)\ntest_set = mnist_test_data.astype('float32') / 255.0\ntest_set","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:32:20.181467Z","iopub.execute_input":"2022-09-15T15:32:20.181726Z","iopub.status.idle":"2022-09-15T15:32:42.031206Z","shell.execute_reply.started":"2022-09-15T15:32:20.181697Z","shell.execute_reply":"2022-09-15T15:32:42.030286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use the obtained model to get predictions","metadata":{}},{"cell_type":"code","source":"results = best_model.cpu()(torch.from_numpy(test_set))\nresults = results.cpu().detach().numpy().argmax(axis=1)\nresults","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:32:42.032336Z","iopub.execute_input":"2022-09-15T15:32:42.032592Z","iopub.status.idle":"2022-09-15T15:32:42.128320Z","shell.execute_reply.started":"2022-09-15T15:32:42.032564Z","shell.execute_reply":"2022-09-15T15:32:42.127402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add the index column and print the predictions into the csv file","metadata":{}},{"cell_type":"code","source":"np.savetxt(\"submission.csv\", \n           np.dstack((np.arange(1, results.size+1),results))[0], \n           \"%d,%d\", \n           header=\"ImageId,Label\", delimiter=\",\", comments=\"\")","metadata":{"execution":{"iopub.status.busy":"2022-09-15T15:32:42.129632Z","iopub.execute_input":"2022-09-15T15:32:42.129911Z","iopub.status.idle":"2022-09-15T15:32:42.193191Z","shell.execute_reply.started":"2022-09-15T15:32:42.129881Z","shell.execute_reply":"2022-09-15T15:32:42.192205Z"},"trusted":true},"execution_count":null,"outputs":[]}]}