{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:55.161321Z","iopub.execute_input":"2022-06-08T01:15:55.162165Z","iopub.status.idle":"2022-06-08T01:15:55.171067Z","shell.execute_reply.started":"2022-06-08T01:15:55.162123Z","shell.execute_reply":"2022-06-08T01:15:55.169514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optim.lr_scheduler.ExponentialLR","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data set","metadata":{}},{"cell_type":"code","source":"print(os.listdir(\"../input/digit-recognizer\"))","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:55.173230Z","iopub.execute_input":"2022-06-08T01:15:55.173879Z","iopub.status.idle":"2022-06-08T01:15:55.196240Z","shell.execute_reply.started":"2022-06-08T01:15:55.173830Z","shell.execute_reply":"2022-06-08T01:15:55.195316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data = pd.read_csv(\"../input/digit-recognizer/train.csv\")\nprint(training_data.shape)\nprint(training_data.columns)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:55.198228Z","iopub.execute_input":"2022-06-08T01:15:55.198575Z","iopub.status.idle":"2022-06-08T01:15:57.947997Z","shell.execute_reply.started":"2022-06-08T01:15:55.198543Z","shell.execute_reply":"2022-06-08T01:15:57.946828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_full = torch.Tensor(training_data.drop(columns=['label']).values).type(torch.FloatTensor)\ny_train_full = torch.Tensor(training_data['label'].values).type(torch.LongTensor)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:57.950042Z","iopub.execute_input":"2022-06-08T01:15:57.950501Z","iopub.status.idle":"2022-06-08T01:15:58.120854Z","shell.execute_reply.started":"2022-06-08T01:15:57.950456Z","shell.execute_reply":"2022-06-08T01:15:58.119761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_full = X_train_full.reshape(42000, 28, 28)\nX_train_full = X_train_full.unsqueeze(1)\nprint(X_train_full.size())","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.123615Z","iopub.execute_input":"2022-06-08T01:15:58.124109Z","iopub.status.idle":"2022-06-08T01:15:58.131063Z","shell.execute_reply.started":"2022-06-08T01:15:58.124064Z","shell.execute_reply":"2022-06-08T01:15:58.129994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_train_full, y_train_full, test_size=0.2, random_state=2022)\nprint(X_train.size())\nprint(X_test.size())\nprint(y_train.size())\nprint(y_test.size())","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.133151Z","iopub.execute_input":"2022-06-08T01:15:58.133652Z","iopub.status.idle":"2022-06-08T01:15:58.602239Z","shell.execute_reply.started":"2022-06-08T01:15:58.133558Z","shell.execute_reply":"2022-06-08T01:15:58.601223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define Dataset class","metadata":{}},{"cell_type":"code","source":"class mnist(Dataset):\n    \n    def __init__(self, images, labels):\n        self.x = images\n        self.y = labels\n        self.num_images = images.size()[0]\n        \n    def __getitem__(self, index):\n        return self.x[index], self.y[index]\n    \n    def __len__(self):\n        return self.num_images\n\ntrain_dataset = mnist(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.603867Z","iopub.execute_input":"2022-06-08T01:15:58.604548Z","iopub.status.idle":"2022-06-08T01:15:58.612859Z","shell.execute_reply.started":"2022-06-08T01:15:58.604500Z","shell.execute_reply":"2022-06-08T01:15:58.611406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_dataset.__len__())","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.614696Z","iopub.execute_input":"2022-06-08T01:15:58.615065Z","iopub.status.idle":"2022-06-08T01:15:58.628324Z","shell.execute_reply.started":"2022-06-08T01:15:58.615032Z","shell.execute_reply":"2022-06-08T01:15:58.627459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define NN Structure","metadata":{}},{"cell_type":"code","source":"class Network(nn.Module):\n    def __init__(self, kernels, filters, cl_out_size, linear_dims):\n        super(Network, self).__init__()\n        self.cl_out_size = cl_out_size\n        self.conv1 = nn.Conv2d(1, filters[0], kernel_size=kernels[0], stride=1, padding=0)\n        self.activation_1 = nn.ReLU()\n        self.conv2 = nn.Conv2d(filters[0], filters[1], kernel_size=kernels[1], stride=1, padding=0)\n        self.activation_2 = nn.ReLU()\n        self.linear_1 = nn.Linear(cl_out_size, linear_dims[0])\n        self.activation_3 = nn.ReLU()\n        self.linear_2 = nn.Linear(linear_dims[0], linear_dims[1])\n        self.activation_4 = nn.ReLU()\n        self.linear_3 = nn.Linear(linear_dims[1], 10)\n        self.activation_5 = nn.ReLU()\n        \n    def forward(self, input):\n        out = self.conv1(input)\n        out = self.activation_1(out)\n        out = self.conv2(out)\n        out = self.activation_2(out)\n        out = out.view(-1, self.cl_out_size)\n        out = self.linear_1(out)\n        out = self.activation_3(out)\n        out = self.linear_2(out)\n        out = self.activation_4(out)\n        out = self.linear_3(out)\n        out = self.activation_5(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.629551Z","iopub.execute_input":"2022-06-08T01:15:58.630587Z","iopub.status.idle":"2022-06-08T01:15:58.645682Z","shell.execute_reply.started":"2022-06-08T01:15:58.630544Z","shell.execute_reply":"2022-06-08T01:15:58.644717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Choose Hyperparameters","metadata":{}},{"cell_type":"code","source":"# These are the hyperparameters\n\nkernel_sizes = [4,4]\nnum_filters = [32, 64]\n\nlinear_layer_dims = [512, 128]\n\nbatch= 1000\nnum_epochs = 4\n\nlearning_rate = 0.005","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.647053Z","iopub.execute_input":"2022-06-08T01:15:58.647971Z","iopub.status.idle":"2022-06-08T01:15:58.666504Z","shell.execute_reply.started":"2022-06-08T01:15:58.647902Z","shell.execute_reply":"2022-06-08T01:15:58.665270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# determine output size of convolutional layers\ndef conv_layer_out_size(input_dim, kernel_sizes, num_filters):\n    out_dim = input_dim\n    for k in kernel_sizes:\n        out_dim -= k-1\n    return out_dim*out_dim*num_filters[-1]\n\nout_size = conv_layer_out_size(28, kernel_sizes, num_filters)\nprint(out_size)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.669750Z","iopub.execute_input":"2022-06-08T01:15:58.670309Z","iopub.status.idle":"2022-06-08T01:15:58.681243Z","shell.execute_reply.started":"2022-06-08T01:15:58.670261Z","shell.execute_reply":"2022-06-08T01:15:58.680326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make instance of model\nmodel = Network(kernel_sizes, num_filters, out_size, linear_layer_dims)\n\n# choose loss function\ncriterion = nn.CrossEntropyLoss()\n\n# choose optimizer\noptimizer = optim.SGD(model.parameters(), lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.682356Z","iopub.execute_input":"2022-06-08T01:15:58.683176Z","iopub.status.idle":"2022-06-08T01:15:58.807929Z","shell.execute_reply.started":"2022-06-08T01:15:58.683135Z","shell.execute_reply":"2022-06-08T01:15:58.807036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define a training loop\ndlr = DataLoader(train_dataset, batch_size=batch, shuffle=True)\n\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(dlr):\n        \n        # forward pass\n        y_pred = model(images)\n        # calculate loss\n        loss = criterion(y_pred, labels)\n        # calculate gradients\n        loss.backward()\n        # update parameters\n        optimizer.step()\n        \n        # clear out gradients for next run\n        optimizer.zero_grad()\n        \n        if i % 1 == 0:\n            print(f\"epoch: {epoch}, batch: {i}, loss: {loss.item()}\")\n        \n        \nprint(f\"epoch: {epoch}, batch: {i}, loss: {loss.item()}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:15:58.835586Z","iopub.execute_input":"2022-06-08T01:15:58.836099Z","iopub.status.idle":"2022-06-08T01:21:31.112740Z","shell.execute_reply.started":"2022-06-08T01:15:58.836065Z","shell.execute_reply":"2022-06-08T01:21:31.111747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    y_test_predictions = model(X_test)\n    accuracy = criterion(y_test_predictions, y_test)\n    print(f\"prediciton loss on test set: {accuracy}\")","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:21:31.114148Z","iopub.execute_input":"2022-06-08T01:21:31.115002Z","iopub.status.idle":"2022-06-08T01:21:39.019789Z","shell.execute_reply.started":"2022-06-08T01:21:31.114952Z","shell.execute_reply":"2022-06-08T01:21:39.018977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_array = y_test_predictions.detach().numpy()\nvector_of_predictions = np.argmax(prediction_array, axis=1)\ntest_numpy_array = y_test.detach().numpy()\na = test_numpy_array == vector_of_predictions\nnp.mean(a)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T01:21:39.021177Z","iopub.execute_input":"2022-06-08T01:21:39.021746Z","iopub.status.idle":"2022-06-08T01:21:39.030193Z","shell.execute_reply.started":"2022-06-08T01:21:39.021709Z","shell.execute_reply":"2022-06-08T01:21:39.029356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# These are the hyperparameters\n\nkernel_sizes = [4,4]\nnum_filters = [32, 64]\n\nlinear_layer_dims = [512, 128]\n\nbatch= 1000\nnum_epochs = 4\n\nlearning_rate = 0.005","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training_loop(hyperparameters, dataset, num_epochs):\n    # creates a network object\n    mdl = Network(hyperparameters[\"kern\"], hyperparameters[\"fil\"],  hyperparameters[\"cl_out_size\"], hyperparameters[\"lin\"])\n    \n    # choose loss function\n    criterion = nn.CrossEntropyLoss()\n\n    # choose optimizer\n    optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n    \n    # make dataloader\n    dlr = DataLoader(dataset, batch_size=hyperparameters[\"bat\"], shuffle=True)\n    \n    # runs training loop\n    for epoch in range(num_epochs):\n        for i, (images, labels) in enumerate(dlr):\n        \n            # forward pass\n            y_pred = mdl(images)\n            # calculate loss\n            loss = criterion(y_pred, labels)\n            # calculate gradients\n            loss.backward()\n            # update parameters\n            optimizer.step()\n        \n            # clear out gradients for next run\n            optimizer.zero_grad()\n            \n            if i % 10 == 0:\n                print(f\"epoch: {epoch}, batch: {i}, loss: {loss.item()}\")\n        \n    \n    print(f\"epoch: {epoch}, batch: {i}, loss: {loss.item()}\")\n    # records performance of trained model\n    ## find test loss\n    with torch.no_grad():\n        y_test_predictions = model(X_test)\n        test_loss = criterion(y_test_predictions, y_test)\n        print(f\"prediciton loss on test set: {test_loss}\")\n    ## find test accuracy\n    prediction_array = y_test_predictions.detach().numpy()\n    vector_of_predictions = np.argmax(prediction_array, axis=1)\n    test_numpy_array = y_test.detach().numpy()\n    acc = test_numpy_array == vector_of_predictions\n    acc = np.mean(acc)\n    # returns performance statistics, and the hyperparameters\n    ...\n    return (test_loss, acc)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T02:15:52.725573Z","iopub.execute_input":"2022-06-08T02:15:52.725992Z","iopub.status.idle":"2022-06-08T02:15:52.738160Z","shell.execute_reply.started":"2022-06-08T02:15:52.725960Z","shell.execute_reply":"2022-06-08T02:15:52.737194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make hyperparameter testing space\nkernels = [4, 5]\nfilters = [32, 64]\nlinear_dims = [128]\nbatch_numbers = [1000]\nlearning_rates = [0.005]\nnum_epochs = 1\n\n# create storage for output\nresults = {}\ncounter = 0\n\n# make dataset\ntrain_dataset = mnist(X_train, y_train)\n\nfor k1 in kernels:\n    for k2 in kernels:\n        for f1 in filters:\n            for f2 in filters:\n                for l1 in linear_dims:\n                    for l2 in linear_dims:\n                        for b in batch_numbers:\n                            for r in learning_rates:\n                                hyperparameters = {\n                                    \"kern\": [k1, k2],\n                                    \"fil\": [f1, f2],\n                                    \"lin\": [l1, l2],\n                                    \"bat\": b,\n                                    \"lr\": r\n                                }\n                                cl_out_size = conv_layer_out_size(28, hyperparameters[\"kern\"], hyperparameters[\"fil\"])\n                                hyperparameters[\"cl_out_size\"] = cl_out_size\n                                output = run_training_loop(hyperparameters, train_dataset, num_epochs)\n                                test_results = {\"hyperparams\": hyperparameters, \"performance\": output}\n                                results[counter] = test_results","metadata":{"execution":{"iopub.status.busy":"2022-06-08T02:21:19.284184Z","iopub.execute_input":"2022-06-08T02:21:19.284598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(results[0][\"performance\"])","metadata":{"execution":{"iopub.status.busy":"2022-06-08T02:17:32.146312Z","iopub.execute_input":"2022-06-08T02:17:32.146826Z","iopub.status.idle":"2022-06-08T02:17:32.154734Z","shell.execute_reply.started":"2022-06-08T02:17:32.146781Z","shell.execute_reply":"2022-06-08T02:17:32.153628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}