{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"__author__: \"Ahmed Bahnasy\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !mkdir \"../output\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_gpu = torch.cuda.is_available()\nuse_gpu","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the data from the csv file\ndf_train_path = \"/kaggle/input/digit-recognizer/train.csv\"\ndf_test_path = \"/kaggle/input/digit-recognizer/test.csv\"\nX_train = pd.read_csv(df_train_path)\nX_test = pd.read_csv(df_test_path)\n# separate into two variables, features and labels\ny_train = X_train['label']\nX_train = X_train.drop('label', axis =1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set_style(\"darkgrid\")\nplt.figure(figsize=(12,8))\nsns.countplot(y_train)\ny_train.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show sample id\nsample_id = 50\nplt.imshow(X_train.loc[sample_id].values.reshape(28,28))\n# plt.imshow(X_train.loc[4].reshape(28,28))\nplt.title(str(y_train[sample_id]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# normailize the data\nX_train /= 255.0\nX_test /= 255.0\n\n# from df to numpy\nX_train = X_train.values\ny_train = y_train.values\nX_test = X_test.values\n\n# split into test and validation\nfrom sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state = 2)\n\n# move to tensor\nX_train = torch.from_numpy(X_train).type(torch.FloatTensor)\nX_val = torch.from_numpy(X_val).type(torch.FloatTensor)\ny_train = torch.from_numpy(y_train).type(torch.LongTensor)\ny_val = torch.from_numpy(y_val).type(torch.LongTensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch_size, epoch and iteration\nbatch_size = 100\nnum_epochs = 20\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create pytorch loaders\ntrain = torch.utils.data.TensorDataset(X_train,y_train)\nval = torch.utils.data.TensorDataset(X_val,y_val)\n# data loader\ntrain_loader = DataLoader(train, batch_size = batch_size, shuffle = False)\nval_loader = DataLoader(val, batch_size = batch_size, shuffle = False)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class LeNet(nn.Module):\n    def __init__(self):\n        super(LeNet,self).__init__()\n        # Network architecture\n        self.conv1 = nn.Conv2d(1,6,(5,5), padding=2)\n        self.conv2 = nn.Conv2d(6,16,(5,5))\n        self.fc1 = nn.Linear(16*5*5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84,10)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, (2,2))\n        x = self.conv2(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, (2,2))\n        \n        # flatten the input\n        shape = x.size()[1:]\n        features = 1\n        for s in shape:\n            features *= s\n        x = x.view(-1, features)\n        \n        \n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        x = F.relu(x)\n        x = self.fc3(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create CNN Model\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super(CNNModel, self).__init__()\n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)\n        self.relu1 = nn.ReLU()\n        # Max pool 1\n        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)\n        self.relu2 = nn.ReLU()\n        # Max pool 2\n        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n        # Fully connected 1\n        self.fc1 = nn.Linear(32 * 4 * 4, 10) \n    def forward(self, x):\n        # Convolution 1\n        out = self.cnn1(x)\n        out = self.relu1(out)\n        # Max pool 1\n        out = self.maxpool1(out)\n        # Convolution 2 \n        out = self.cnn2(out)\n        out = self.relu2(out)\n        # Max pool 2 \n        out = self.maxpool2(out)\n        # flatten\n        out = out.view(out.size(0), -1)\n        # Linear function (readout)\n        out = self.fc1(out)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Lenet with dropout\n\nclass LeNet_dropout(nn.Module):\n    def __init__(self):\n        super(LeNet_dropout, self).__init__()\n        self.conv1 = nn.Conv2d(1,6,(5,5), padding=2)\n        self.dropout1 = nn.Dropout2d(p=0.2)\n        self.conv2 = nn.Conv2d(6,16,(5,5))\n        self.dropout2 = nn.Dropout2d(p=0.2)\n        self.fc1 = nn.Linear(16*5*5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84,10)\n    \n    def forward(self, x):\n        x = self.conv1(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, (2,2))\n        x = self.dropout1(x)\n        x = self.conv2(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, (2,2))\n        x = self.dropout2(x)\n        \n        # flatten the input\n        shape = x.size()[1:]\n        features = 1\n        for s in shape:\n            features *= s\n        x = x.view(-1, features)\n        \n        \n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        x = F.relu(x)\n        x = self.fc3(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Initiation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# lenet SGD wit momentum\nmodel = LeNet()\nif use_gpu:\n\tmodel = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = LeNet_dropout()\nif use_gpu:\n\tmodel = net.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CNNModel()\nif use_gpu:\n\tmodel = net.cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss functions Initiation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimizer initiation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.SGD(model.parameters(), lr=0.001)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"count = 0\nloss_list = []\nval_loss_list = []\niteration_list = []\naccuracy_list = []\nfor epoch in range(num_epochs):\n    for images, labels in (train_loader):\n        train_batch = Variable(images.view(batch_size, 1, 28, 28), requires_grad = True)\n        labels_batch = Variable(labels, requires_grad = False)\n        # move mini_batch to gpu\n        if use_gpu:\n            train_batch  = train_batch.cuda()\n            labels_batch = labels_batch.cuda()\n        # clear gradients\n        optimizer.zero_grad()\n        \n        # forward propagation\n        outputs = model(train_batch)\n        \n        # calculate loss\n        loss = criterion(outputs, labels_batch)\n        #backprobagation\n        loss.backward()\n        # update weights\n        optimizer.step()\n        \n        count +=1\n        \n        # evaluate on the validation data every 50 iterations\n        if count % 50 == 0:\n            # calculate accuracy\n            correct = 0\n            total = 0\n            for images, labels in val_loader:\n                val_batch = Variable(images.view(-1,1,28,28), requires_grad = False)\n                labels_batch = Variable(labels, requires_grad = False)\n                if use_gpu:\n                    val_batch  = val_batch.cuda()\n                    labels_batch = labels_batch.cuda()\n                outputs = model(val_batch)\n                pred = torch.max(outputs.data, 1)[1]\n                \n                # calculate loss\n                val_loss = criterion(outputs, labels_batch)\n                \n    \n                total += len(labels)\n    \n                correct += (pred == labels_batch).sum()\n                \n            accuracy = (correct /float(total)) * 100\n\n            loss_list.append(loss.data)\n            val_loss_list.append(val_loss.data)\n            iteration_list.append(count)\n            accuracy_list.append(accuracy)\n            \n        if count % 500 == 0:\n            print(\"Iteration: {}, Loss: {}, Accuracy: {}\".format(count, loss.data, accuracy))\n                \n        \n        \n        # print insights every 500 iterations\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualization loss \nplt.figure(figsize=(15,8))\nplt.plot(iteration_list,loss_list, label=\"training\")\nplt.plot(iteration_list,val_loss_list, label='validation')\nplt.xlabel(\"Number of iteration\")\nplt.ylabel(\"Loss\")\nplt.title(\"CNN: Loss vs Number of iteration\")\nplt.legend()\nplt.show()\n\n\nplt.figure(figsize=(15,8))\n# visualization accuracy \nplt.plot(iteration_list,accuracy_list,color = \"red\")\nplt.xlabel(\"Number of iteration\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"CNN: Accuracy vs Number of iteration\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the output file container\noutput_file = np.ndarray(shape=(n_test_samples,2), dtype = int)\n\nfor test_idx in range(n_test_samples):\n    test_sample = X_test[test_idx].clone().unsqueeze(dim = 1)\n    test_sample = test_sample.type(torch.FloatTensor)\n    if use_gpu:\n        test_sample = test_sample.cuda()\n    pred = net(test_sample)\n    # get the index of the max class\n    _ , pred = torch.max(pred,1)\n    output_file[test_idx][0] = test_idx+1\n    output_file[test_idx][1] = pred\n    \n    if test_idx % 1000 ==0:\n        print(f\"testing sample #{test_idx}\")\n        \n\n\nsubmission = pd.DataFrame(output_file, dtype=int, columns=['ImageId', 'Label'])\n\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity check\nsample = 460\nplt.imshow(X_test[sample].reshape(28,28))\n\nprint(output_file[sample][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Generating output file\\n\")\nsubmission.to_csv('pytorch_LeNet.csv', index=False, header=True)\n# from IPython.display import FileLink\n# FileLink(r'pytorch_LeNet.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"experiment with bias = true in conv2d function","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}