{"cells":[{"metadata":{"_uuid":"971dad4d-c433-4bc9-ae59-20e6bac3a473","_cell_guid":"fd0fcc8e-5a85-4267-bc1f-02e1a8744c6d","trusted":true},"cell_type":"code","source":"# Classifying Digits using Simple ML models\n# Necessary Imports\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import accuracy_score\n\n# Exploring files in the Input Directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Step 1: Convert the data into usable format\n# pandas is used to read the contents of csv into a dataframe\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Retrieve the features and label columns into a separate numpy arrays\nfeatures = train[train.columns[1:]].values\nlabel = train.label.values\n\nprint(type(features))\nprint(type(label))\n\nprint(features.shape)\nprint(label.shape)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 1: **\n\nLogistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.simplefilter(\"ignore\")\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137)\nacc = 0.0\nglobal best_model\n\nfor jj, (train_index, val_index) in enumerate(kf.split(features)):\n    print(\"Fitting fold\", jj+1)\n    train_features = features[train_index]\n    train_target = label[train_index]\n    \n    val_features = features[val_index]\n    val_target = label[val_index]\n    \n    model = LogisticRegression(C=20, solver='lbfgs', multi_class='multinomial')\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)\n    fold_acc=accuracy_score(val_target, np.argmax(val_pred, axis=1))\n    print(\"Fold accuracy:\", accuracy_score(val_target, np.argmax(val_pred, axis=1)))\n    #test_preds += model.predict_proba(test)/n_splits\n    if(fold_acc>acc):\n        acc = fold_acc\n        best_model = model\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nprint(test.columns)\n#Retrieve the features and label columns into a separate numpy arrays\ntest_features = test[test.columns[0:]].values\ntest_pred = model.predict_proba(test_features)\npredict = np.argmax(test_pred, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmission['Label'] = predict\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 2: **\n\nLogistic Regression with STOCHASTIC GRADIENT DESCENT "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137)\nacc = 0.0\nglobal best_model\n\nfor jj, (train_index, val_index) in enumerate(kf.split(features)):\n    print(\"Fitting fold\", jj+1)\n    train_features = features[train_index]\n    train_target = label[train_index]\n    \n    val_features = features[val_index]\n    val_target = label[val_index]\n    \n    # loss = 'hinge' represents linear regression\n    # log loss implement logistic regression\n    model = SGDClassifier(loss='log')\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)\n    fold_acc=accuracy_score(val_target, np.argmax(val_pred, axis=1))\n    print(\"Fold accuracy:\", accuracy_score(val_target, np.argmax(val_pred, axis=1)))\n    #test_preds += model.predict_proba(test)/n_splits\n    if(fold_acc>acc):\n        acc = fold_acc\n        best_model = model\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 3:**\nRandomForest Classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nn_splits = 5\nkf = KFold(n_splits=n_splits, random_state=137)\nacc = 0.0\nglobal best_model\n\nfor jj, (train_index, val_index) in enumerate(kf.split(features)):\n    print(\"Fitting fold\", jj+1)\n    train_features = features[train_index]\n    train_target = label[train_index]\n    \n    val_features = features[val_index]\n    val_target = label[val_index]\n    \n    # max depth serves as an important hyperparameter\n    # When the depth was set with a value of 2, accuracy was down by 60%\n    model = RandomForestClassifier(n_estimators=100, max_depth=10, random_state=0)\n    model.fit(train_features, train_target)\n    val_pred = model.predict_proba(val_features)\n    fold_acc=accuracy_score(val_target, np.argmax(val_pred, axis=1))\n    print(\"Fold accuracy:\", accuracy_score(val_target, np.argmax(val_pred, axis=1)))\n    #test_preds += model.predict_proba(test)/n_splits\n    if(fold_acc>acc):\n        acc = fold_acc\n        best_model = model\n    del train_features, train_target, val_features, val_target\n    gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nprint(test.columns)\n#Retrieve the features and label columns into a separate numpy arrays\ntest_features = test[test.columns[0:]].values\ntest_pred = model.predict_proba(test_features)\npredict = np.argmax(test_pred, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmission['Label'] = predict\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 3:**\nConvolutional Neural Network"},{"metadata":{},"cell_type":"markdown","source":"**Defining a simple convolutional neural network**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nfrom torch import nn\nclass Flatten(nn.Module):\n    def __init__(self):\n        super(Flatten, self).__init__()\n\n    def forward(self, x):\n        return x.view(x.size(0), -1)\n\nclass NN(torch.nn.Module):\n    def __init__(self):\n        super(NN,self).__init__()\n        self.conv1 = torch.nn.Conv2d(1,6,3,padding=1)\n        self.pool1 = torch.nn.MaxPool2d(2)\n        self.conv2 = torch.nn.Conv2d(6,16,5,padding=0)\n        self.pool2 = torch.nn.MaxPool2d(2)\n        self.linear1 = torch.nn.Linear(400,120)\n        self.linear2 = torch.nn.Linear(120,84)\n        self.linear3 = torch.nn.Linear(84,10)\n    def forward(self,x):\n        c1= F.relu(self.conv1(x))\n        s1 = self.pool1(c1)\n        c2 = F.relu(self.conv2(s1))\n        s2 = self.pool2(c2)\n        f  = (Flatten()(s2))\n        f1 = F.relu(self.linear1(f))\n        f2 = F.relu(self.linear2(f1))\n        f3 = self.linear3(f2)\n        return f3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Install Necessary Packages\n!pip install torchsummary","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Selecting the device and initiating the model**\ntorch summary provides a detailed picture on the arrangement of layers in the resulting neural network"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torchsummary import summary\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = NN().to(device)\nsummary(model,(1,28,28))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Reading the contents of the csv file \nReshaping the input from a single dimensional vector to a image of size (1,28,28)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Retrieve the features and label columns into a separate numpy arrays\nfeatures = train[train.columns[1:]].values\nlabel = train.label.values\n\n[rows, columns] = features.shape\nprint(type(features))\nprint(type(label))\n\nprint(features.shape)\nprint(label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = features /255.0\nfeatures = features.reshape(rows,1,28,28)\nprint(features.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Defining the dataset.** A Class which accepts the input images and the labels; can be used by loaders to feed the images to the network in batches"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nclass mnistDataset(Dataset):\n    def __init__(self, images, labels):\n        self.image =  torch.from_numpy(images)\n        self.gt = torch.from_numpy(labels)\n\n    def __len__(self):\n        #print(self.image.shape)\n        #print(self.gt.shape)\n        return self.image.shape[0]\n\n    def __getitem__(self,index):\n        return self.image[index], self.gt[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = mnistDataset(features, label)\nprint(features.shape)\nprint(label.shape)\ntrainLoader = torch.utils.data.DataLoader(dataset, batch_size=5, shuffle=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define the Criterion and Optimizer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim\nCriterion = torch.nn.CrossEntropyLoss(reduction='mean')\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Train the model **\n1. Obtain the model prediction for batch from the dataset\n2. Compare target against prediction\n3. Obtain gradients and adjust weights based on the loss\n4. Repeat for all the batches in the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, Criterion, optimizer, trainLoader):\n    model.train()\n    epoch_list=[]\n    loss_list=[]\n    for epoch in range(100):\n        running_loss = 0\n        for data,target in trainLoader:\n            x = data.to(device)\n            x = x.type(torch.cuda.FloatTensor)\n            y = target.to(device)\n            #Compute model ouput\n            pred = model(x)\n            #print(pred)\n            #print(y)\n            #Compute loss\n            loss = Criterion(pred,y)\n            running_loss +=loss.item()\n            #Optimizer to adjust weights\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n        print('epoch:',epoch,'loss:',running_loss/len(trainLoader))\n        epoch_list.append(epoch)\n        loss_list.append(running_loss/len(trainLoader))\n    return epoch_list,loss_list, model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch_list, loss_list, model = train(model, Criterion, optimizer, trainLoader)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nclass mnisttestDataset(Dataset):\n    def __init__(self, images):\n        self.image =  torch.from_numpy(images)\n\n    def __len__(self):\n        #print(self.image.shape)\n        #print(self.gt.shape)\n        return self.image.shape[0]\n\n    def __getitem__(self,index):\n        return self.image[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n#Retrieve the features and label columns into a separate numpy arrays\nfeatures = test[test.columns[0:]].values\nprint(features.shape)\nfeatures = features /255.0\nfeatures = features.reshape(28000,1,28,28)\n\ndataset = mnisttestDataset(features)\nprint(features.shape)\nprint(label.shape)\ntestLoader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom torch.autograd import Variable\ndef validation(model, optimizer, testLoader, device, Criterion):\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for vinput in testLoader:\n            vinput = Variable(vinput)\n            vinput = vinput.to(device)\n            vinput = vinput.type(torch.cuda.FloatTensor)\n            optimizer.zero_grad()\n            vpredict = model(vinput)\n            predict = F.softmax(vpredict, dim=1)\n            predict = predict.to('cpu')\n            predict = predict.numpy()\n            #print(predict)\n            predictions.append(np.argmax(predict, axis=1))\n            #print(predictions)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = validation(model, optimizer, testLoader, device, Criterion)\npredict_labels = []\n\nfor i in range(28000):\n    predict_labels.append(predict[i][0])\n\n\n\nsubmission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmission['Label'] = predict_labels\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model 4:**\nFinetune Neural Network 'ResNet18 Model'"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.models as models\nimport torch.nn as nn\nimport torch\n#Instantiating ResNet model with pretrained weights\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = models.resnet18(pretrained = True)\nnum_features = model.fc.in_features\n#Modifying the fully connected layer to reduce number of classes from 1000 to 2\nmodel.fc = nn.Linear(num_features, 10)\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Retrieve the features and label columns into a separate numpy arrays\nfeatures = train[train.columns[1:]].values\nlabel = train.label.values\n\n[rows, columns] = features.shape\nfeatures = np.tile(features[:,:,None],[1,1,3])\nprint(type(features))\nprint(type(label))\n\nprint(features.shape)\nprint(label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = features /255.0\nfeatures = features.transpose((0,2,1))\nfeatures = features.reshape(rows,3,28,28)\nprint(features.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nclass mnistDataset(Dataset):\n    def __init__(self, images, labels):\n        self.image =  torch.from_numpy(images)\n        self.gt = torch.from_numpy(labels)\n\n    def __len__(self):\n        #print(self.image.shape)\n        #print(self.gt.shape)\n        return self.image.shape[0]\n\n    def __getitem__(self,index):\n        return self.image[index], self.gt[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = mnistDataset(features, label)\nprint(features.shape)\nprint(label.shape)\ntrainLoader = torch.utils.data.DataLoader(dataset, batch_size=5, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params_to_update = model.parameters()\noptimizer = torch.optim.SGD(params_to_update, lr=0.001, momentum=0.9)\nCriterion = nn.CrossEntropyLoss(reduction='mean')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, Criterion, optimizer, trainLoader):\n    model.train()\n    epoch_list=[]\n    loss_list=[]\n    for epoch in range(10):\n        running_loss = 0\n        for data,target in trainLoader:\n            x = data.to(device)\n            x = x.type(torch.cuda.FloatTensor)\n            y = target.to(device)\n            #Compute model ouput\n            pred = model(x)\n            #print(pred)\n            #print(y)\n            #Compute loss\n            loss = Criterion(pred,y)\n            running_loss +=loss.item()\n            #Optimizer to adjust weights\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n        print('epoch:',epoch,'loss:',running_loss/len(trainLoader))\n        epoch_list.append(epoch)\n        loss_list.append(running_loss/len(trainLoader))\n    return epoch_list,loss_list, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch_list, loss_list, model = train(model, Criterion, optimizer, trainLoader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nclass mnisttestDataset(Dataset):\n    def __init__(self, images):\n        self.image =  torch.from_numpy(images)\n\n    def __len__(self):\n        #print(self.image.shape)\n        #print(self.gt.shape)\n        return self.image.shape[0]\n\n    def __getitem__(self,index):\n        return self.image[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n#Retrieve the features and label columns into a separate numpy arrays\nfeatures = test[test.columns[0:]].values\nprint(features.shape)\nfeatures = features /255.0\nfeatures = np.tile(features[:,:,None],[1,1,3])\nfeatures = features.reshape(28000,3,28,28)\ndataset = mnisttestDataset(features)\nprint(features.shape)\nprint(label.shape)\ntestLoader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\ndef validation(model, optimizer, testLoader, device, Criterion):\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for vinput in testLoader:\n            vinput = Variable(vinput)\n            vinput = vinput.to(device)\n            vinput = vinput.type(torch.cuda.FloatTensor)\n            optimizer.zero_grad()\n            vpredict = model(vinput)\n            predict = F.softmax(vpredict, dim=1)\n            predict = predict.to('cpu')\n            predict = predict.numpy()\n            #print(predict)\n            predictions.append(np.argmax(predict, axis=1))\n            #print(predictions)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = validation(model, optimizer, testLoader, device, Criterion)\npredict_labels = []\n\nfor i in range(28000):\n    predict_labels.append(predict[i][0])\n\nsubmission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsubmission['Label'] = predict_labels\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Reference:**\nhttps://www.kaggle.com/tunguz/mnist-logistic-regression-baseline"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}