{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"KNN for MNIST","metadata":{}},{"cell_type":"markdown","source":"Import libraries","metadata":{}},{"cell_type":"code","source":"#import libraries\nimport cv2\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import accuracy_score,classification_report,confusion_matrix\n\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:06:26.651574Z","iopub.execute_input":"2022-01-08T09:06:26.651941Z","iopub.status.idle":"2022-01-08T09:06:33.011319Z","shell.execute_reply.started":"2022-01-08T09:06:26.651855Z","shell.execute_reply":"2022-01-08T09:06:33.010299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import and load data","metadata":{}},{"cell_type":"code","source":"#load mnist data from Kaggle\ndf_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf_test = pd.read_csv(\"../input/labelled-test-dataset/labelledtest.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:10:43.304168Z","iopub.execute_input":"2022-01-08T09:10:43.304715Z","iopub.status.idle":"2022-01-08T09:10:49.256523Z","shell.execute_reply.started":"2022-01-08T09:10:43.304668Z","shell.execute_reply":"2022-01-08T09:10:49.255762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Preview data","metadata":{}},{"cell_type":"code","source":"#display train dataframe\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:11:48.937378Z","iopub.execute_input":"2022-01-08T09:11:48.937658Z","iopub.status.idle":"2022-01-08T09:11:48.964655Z","shell.execute_reply.started":"2022-01-08T09:11:48.937629Z","shell.execute_reply":"2022-01-08T09:11:48.963918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#display train dataframe\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:11:59.848518Z","iopub.execute_input":"2022-01-08T09:11:59.848931Z","iopub.status.idle":"2022-01-08T09:11:59.865182Z","shell.execute_reply.started":"2022-01-08T09:11:59.848898Z","shell.execute_reply":"2022-01-08T09:11:59.864427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#display test dataframe\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:12:13.611267Z","iopub.execute_input":"2022-01-08T09:12:13.611955Z","iopub.status.idle":"2022-01-08T09:12:13.627394Z","shell.execute_reply.started":"2022-01-08T09:12:13.611917Z","shell.execute_reply":"2022-01-08T09:12:13.626641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test data stat\ndf_test.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:12:21.679925Z","iopub.execute_input":"2022-01-08T09:12:21.680660Z","iopub.status.idle":"2022-01-08T09:12:23.503993Z","shell.execute_reply.started":"2022-01-08T09:12:21.680616Z","shell.execute_reply":"2022-01-08T09:12:23.503168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info() #with label","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:12:36.784224Z","iopub.execute_input":"2022-01-08T09:12:36.784520Z","iopub.status.idle":"2022-01-08T09:12:36.838255Z","shell.execute_reply.started":"2022-01-08T09:12:36.784491Z","shell.execute_reply":"2022-01-08T09:12:36.837372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.info() #missing label added","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:13:13.999154Z","iopub.execute_input":"2022-01-08T09:13:13.999438Z","iopub.status.idle":"2022-01-08T09:13:14.049876Z","shell.execute_reply.started":"2022-01-08T09:13:13.999407Z","shell.execute_reply":"2022-01-08T09:13:14.049007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape #training set shape","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:13:21.865641Z","iopub.execute_input":"2022-01-08T09:13:21.865918Z","iopub.status.idle":"2022-01-08T09:13:21.870863Z","shell.execute_reply.started":"2022-01-08T09:13:21.865887Z","shell.execute_reply":"2022-01-08T09:13:21.870360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.shape #test set shape, missing label column","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:13:27.789064Z","iopub.execute_input":"2022-01-08T09:13:27.789468Z","iopub.status.idle":"2022-01-08T09:13:27.794383Z","shell.execute_reply.started":"2022-01-08T09:13:27.789438Z","shell.execute_reply":"2022-01-08T09:13:27.793363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pre-processing","metadata":{}},{"cell_type":"code","source":"X_train = df_train.drop('label', axis = 1) #drop label column\nX_train = X_train.values.astype('float64') #convert to float 64\nX_train #show","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:14:26.606750Z","iopub.execute_input":"2022-01-08T09:14:26.607205Z","iopub.status.idle":"2022-01-08T09:14:26.815238Z","shell.execute_reply.started":"2022-01-08T09:14:26.607164Z","shell.execute_reply":"2022-01-08T09:14:26.814322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = df_train['label'] #assign the label column to y\ny_train = y_train.values.astype('int64') #convert to int 64\ny_train #show","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:14:39.554592Z","iopub.execute_input":"2022-01-08T09:14:39.554905Z","iopub.status.idle":"2022-01-08T09:14:39.562502Z","shell.execute_reply.started":"2022-01-08T09:14:39.554873Z","shell.execute_reply":"2022-01-08T09:14:39.561668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = df_test.drop('label', axis = 1) #assign to the whole test set since label column is missing\nX_test = X_test.values.astype('float64') #convert to float 64\nX_test","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:14:47.289144Z","iopub.execute_input":"2022-01-08T09:14:47.289811Z","iopub.status.idle":"2022-01-08T09:14:47.419488Z","shell.execute_reply.started":"2022-01-08T09:14:47.289774Z","shell.execute_reply":"2022-01-08T09:14:47.418649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = df_test['label'] #assign the label column to y\ny_test = y_test.values.astype('int64') #convert to int 64\ny_test","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:14:53.581468Z","iopub.execute_input":"2022-01-08T09:14:53.581746Z","iopub.status.idle":"2022-01-08T09:14:53.587263Z","shell.execute_reply.started":"2022-01-08T09:14:53.581718Z","shell.execute_reply":"2022-01-08T09:14:53.586447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resX_train = X_train.reshape(X_train.shape[0], 28, 28) #reshape to 28 x 28 (sqrt of 784)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:23.035041Z","iopub.execute_input":"2022-01-08T09:16:23.035311Z","iopub.status.idle":"2022-01-08T09:16:23.039699Z","shell.execute_reply.started":"2022-01-08T09:16:23.035274Z","shell.execute_reply":"2022-01-08T09:16:23.039077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resX_test = X_test.reshape(X_test.shape[0], 28, 28) #reshape to 28 x 28 (sqrt of 784)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:26.038393Z","iopub.execute_input":"2022-01-08T09:16:26.038835Z","iopub.status.idle":"2022-01-08T09:16:26.042910Z","shell.execute_reply.started":"2022-01-08T09:16:26.038801Z","shell.execute_reply":"2022-01-08T09:16:26.042156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train image sampling\ndigit = 10\nfig, axes = plt.subplots(1, digit, figsize=(28,28))\n\nfor i in range(0, digit):\n  trainsamp = resX_train[y_train == i][0]\n  axes[i].imshow(tf.squeeze(trainsamp), cmap = 'gray')\n  axes[i].set_title(\"True: {}\".format(i), fontsize = 12)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:28.386033Z","iopub.execute_input":"2022-01-08T09:16:28.386304Z","iopub.status.idle":"2022-01-08T09:16:29.780152Z","shell.execute_reply.started":"2022-01-08T09:16:28.386276Z","shell.execute_reply":"2022-01-08T09:16:29.779336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Deskew","metadata":{}},{"cell_type":"code","source":"#deskew function\ndef deskew(im):\n  immom = cv2.moments(im) #image moments\n  if abs(immom['mu02']) < 0: #prevent zero divi\n    return im.copy()\n    \n  ratio = immom['mu11']/immom['mu02'] #ratio between 2 central 2nd order moments\n  mat = np.float32([[1, ratio, -0.5*28*ratio], [0, 1, 0]]) #2x3 translate + shear matrix\n\n  im = cv2.warpAffine(im, mat, (28, 28), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_AREA) #linear reverse mapping\n\n  return im","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:56.440549Z","iopub.execute_input":"2022-01-08T09:16:56.441346Z","iopub.status.idle":"2022-01-08T09:16:56.448126Z","shell.execute_reply.started":"2022-01-08T09:16:56.441308Z","shell.execute_reply":"2022-01-08T09:16:56.447365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#equate\ndskewresX_train = resX_train\ndskewresX_test = resX_test","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:22:09.888584Z","iopub.execute_input":"2022-01-08T09:22:09.888947Z","iopub.status.idle":"2022-01-08T09:22:09.894051Z","shell.execute_reply.started":"2022-01-08T09:22:09.888911Z","shell.execute_reply":"2022-01-08T09:22:09.893145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#deskew train\nfor i in range(dskewresX_train.shape[0]):\n  dskewresX_train[i] = deskew(dskewresX_train[i]) #deskew train data","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:22:16.952205Z","iopub.execute_input":"2022-01-08T09:22:16.952786Z","iopub.status.idle":"2022-01-08T09:22:18.455220Z","shell.execute_reply.started":"2022-01-08T09:22:16.952749Z","shell.execute_reply":"2022-01-08T09:22:18.454170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#deskew test\nfor i in range(dskewresX_test.shape[0]):\n  dskewresX_test[i] = deskew(dskewresX_test[i]) #deskew test data","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:22:23.104518Z","iopub.execute_input":"2022-01-08T09:22:23.104818Z","iopub.status.idle":"2022-01-08T09:22:24.118920Z","shell.execute_reply.started":"2022-01-08T09:22:23.104781Z","shell.execute_reply":"2022-01-08T09:22:24.117902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#deskew train image sampling\ndigit = 10\nfig, axes = plt.subplots(1, digit, figsize=(28,28))\n\nfor i in range(0, digit):\n  dskewtrainsamp = dskewresX_train[y_train == i][0]\n  axes[i].imshow(tf.squeeze(dskewtrainsamp), cmap = 'gray')\n  axes[i].set_title(\"True: {}\".format(i), fontsize = 12)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:22:28.078960Z","iopub.execute_input":"2022-01-08T09:22:28.079266Z","iopub.status.idle":"2022-01-08T09:22:29.538703Z","shell.execute_reply.started":"2022-01-08T09:22:28.079230Z","shell.execute_reply":"2022-01-08T09:22:29.537684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Morphology","metadata":{}},{"cell_type":"code","source":"kernel = np.matrix(('1 1'), np.uint8) #erosion kernel","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:22:49.288817Z","iopub.execute_input":"2022-01-08T09:22:49.289096Z","iopub.status.idle":"2022-01-08T09:22:49.293309Z","shell.execute_reply.started":"2022-01-08T09:22:49.289067Z","shell.execute_reply":"2022-01-08T09:22:49.292373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for HoGKNNseries(deskew+erosion), use dskewresX_train\n#for HoGKNNparallel(deskew+erosion), use resX_train\neroresX_train = cv2.erode(resX_train, kernel, iterations=1) #erosion of train data\n#for HoGKNNseries(deskew+erosion), use dskewresX_test\n#for HoGKNNparallel(deskew+erosion), use resX_test\neroresX_test = cv2.erode(resX_test, kernel, iterations=1) #erosion of test data","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:22:59.355265Z","iopub.execute_input":"2022-01-08T09:22:59.355844Z","iopub.status.idle":"2022-01-08T09:22:59.942573Z","shell.execute_reply.started":"2022-01-08T09:22:59.355776Z","shell.execute_reply":"2022-01-08T09:22:59.941832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(eroresX_train) #length","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#erosion train image sampling\ndigit = 10\nfig, axes = plt.subplots(1, digit, figsize=(28,28))\nfor i in range(0, digit):\n  erotrainsamp = eroresX_train[y_train == i][0]\n  axes[i].imshow(tf.squeeze(erotrainsamp), cmap = 'gray')\n  axes[i].set_title(\"True: {}\".format(i), fontsize = 12)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:23:14.347500Z","iopub.execute_input":"2022-01-08T09:23:14.347793Z","iopub.status.idle":"2022-01-08T09:23:15.572690Z","shell.execute_reply.started":"2022-01-08T09:23:14.347762Z","shell.execute_reply":"2022-01-08T09:23:15.571768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Concatenate","metadata":{}},{"cell_type":"code","source":"#for HoGKNN(base), use resX_train\n#for HoGKNN(deskew), use dskewresX_train\n#for HoGKNNseries(deskew+erosion), use eroresX_train\n#for HoGKNNparallel(deskew+erosion), use np.concatenate((dskewresX_train, eroresX_train),axis = 0)\nnewX_train = dskewresX_train\n#for HoGKNN(base), use resX_test\n#for HoGKNN(deskew), use dskewresX_test\n#for HoGKNNseries(deskew+erosion), use eroresX_test\n#for HoGKNNparallel(deskew+erosion), use np.concatenate((dskewresX_test, eroresX_test),axis = 0)\nnewX_test = dskewresX_test","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:30:39.188885Z","iopub.execute_input":"2022-01-08T09:30:39.189759Z","iopub.status.idle":"2022-01-08T09:30:39.194092Z","shell.execute_reply.started":"2022-01-08T09:30:39.189717Z","shell.execute_reply":"2022-01-08T09:30:39.193226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#new train data length\nlen(newX_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:30:48.597442Z","iopub.execute_input":"2022-01-08T09:30:48.598299Z","iopub.status.idle":"2022-01-08T09:30:48.603665Z","shell.execute_reply.started":"2022-01-08T09:30:48.598257Z","shell.execute_reply":"2022-01-08T09:30:48.603035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#new test data length\nlen(newX_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:30:52.545166Z","iopub.execute_input":"2022-01-08T09:30:52.546014Z","iopub.status.idle":"2022-01-08T09:30:52.551297Z","shell.execute_reply.started":"2022-01-08T09:30:52.545960Z","shell.execute_reply":"2022-01-08T09:30:52.550749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for HoGKNN(base), use y_train\n#for HoGKNN(deskew), use y_train\n#for HoGKNNseries(deskew+erosion), use y_train\n#for HoGKNNparallel(deskew+erosion), use np.concatenate((y_train, y_train),axis = 0)\nnewy_train = y_train\n#for HoGKNN(base), use y_test\n#for HoGKNN(deskew), use y_test\n#for HoGKNNseries(deskew+erosion), use y_test\n#for HoGKNNparallel(deskew+erosion), use np.concatenate((y_test, y_test),axis = 0)\nnewy_test = y_test","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:30:58.488782Z","iopub.execute_input":"2022-01-08T09:30:58.489118Z","iopub.status.idle":"2022-01-08T09:30:58.493699Z","shell.execute_reply.started":"2022-01-08T09:30:58.489086Z","shell.execute_reply":"2022-01-08T09:30:58.492736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"HoG","metadata":{}},{"cell_type":"code","source":"#hog parameters\nwinSize = (28, 28) #same as image size\nblockSize = (12, 12) #12 x 12 block\nblockStride = (4, 4) #4 by 4\ncellSize = (12, 12) #12 x 12 cell\nnbins = 9 #9 bins\nsignedGradients = True #0-180\nderivAperture = 1 #default\nwinSigma = -1.0 #default\nhistogramNormType = 0 #none\nL2HysThreshold = 0.2 #default\ngammaCorrection = 1 #default\nnlevels = 64 #default","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:31:19.877804Z","iopub.execute_input":"2022-01-08T09:31:19.878576Z","iopub.status.idle":"2022-01-08T09:31:19.883858Z","shell.execute_reply.started":"2022-01-08T09:31:19.878537Z","shell.execute_reply":"2022-01-08T09:31:19.883046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#hog descriptors\nhog = cv2.HOGDescriptor(winSize, blockSize, blockStride, cellSize, nbins, derivAperture, winSigma, histogramNormType, L2HysThreshold, gammaCorrection, nlevels, signedGradients)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:31:29.500012Z","iopub.execute_input":"2022-01-08T09:31:29.500581Z","iopub.status.idle":"2022-01-08T09:31:29.506817Z","shell.execute_reply.started":"2022-01-08T09:31:29.500537Z","shell.execute_reply":"2022-01-08T09:31:29.505952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train data formatting\nui8resX_train = np.uint8(newX_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:31:50.213952Z","iopub.execute_input":"2022-01-08T09:31:50.214353Z","iopub.status.idle":"2022-01-08T09:31:50.257823Z","shell.execute_reply.started":"2022-01-08T09:31:50.214321Z","shell.execute_reply":"2022-01-08T09:31:50.256949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for train descriptors\ntrain_descriptors = []\nfor i in range(ui8resX_train.shape[0]):\n    traindescrip = hog.compute(ui8resX_train[i]) #hog for train data\n    train_descriptors.append(traindescrip) #append to train decriptors list","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:31:59.656015Z","iopub.execute_input":"2022-01-08T09:31:59.656286Z","iopub.status.idle":"2022-01-08T09:32:01.150148Z","shell.execute_reply.started":"2022-01-08T09:31:59.656254Z","shell.execute_reply":"2022-01-08T09:32:01.149320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#length of train descriptors\nlen(train_descriptors[0])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:32:08.130343Z","iopub.execute_input":"2022-01-08T09:32:08.130584Z","iopub.status.idle":"2022-01-08T09:32:08.135810Z","shell.execute_reply.started":"2022-01-08T09:32:08.130558Z","shell.execute_reply":"2022-01-08T09:32:08.135109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train data resize\ntrain_descriptors = np.resize(train_descriptors, (ui8resX_train.shape[0], 225))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:32:20.916723Z","iopub.execute_input":"2022-01-08T09:32:20.917505Z","iopub.status.idle":"2022-01-08T09:32:20.982641Z","shell.execute_reply.started":"2022-01-08T09:32:20.917464Z","shell.execute_reply":"2022-01-08T09:32:20.981911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test data formatting\nui8resX_test = np.uint8(newX_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:32:38.403509Z","iopub.execute_input":"2022-01-08T09:32:38.404064Z","iopub.status.idle":"2022-01-08T09:32:38.427213Z","shell.execute_reply.started":"2022-01-08T09:32:38.404028Z","shell.execute_reply":"2022-01-08T09:32:38.426663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_descriptors = []\nfor i in range(ui8resX_test.shape[0]):\n    testdescrip = hog.compute(ui8resX_test[i]) # compute the HOG features\n    test_descriptors.append(testdescrip) # append it to the test descriptors list","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:32:46.024366Z","iopub.execute_input":"2022-01-08T09:32:46.024814Z","iopub.status.idle":"2022-01-08T09:32:46.990252Z","shell.execute_reply.started":"2022-01-08T09:32:46.024784Z","shell.execute_reply":"2022-01-08T09:32:46.989149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#length of test descriptors\nlen(test_descriptors[0])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:32:52.498192Z","iopub.execute_input":"2022-01-08T09:32:52.498471Z","iopub.status.idle":"2022-01-08T09:32:52.503756Z","shell.execute_reply.started":"2022-01-08T09:32:52.498439Z","shell.execute_reply":"2022-01-08T09:32:52.503002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test descriptor resize\ntest_descriptors = np.resize(test_descriptors, (ui8resX_test.shape[0], 225))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:32:58.775277Z","iopub.execute_input":"2022-01-08T09:32:58.775968Z","iopub.status.idle":"2022-01-08T09:32:58.815635Z","shell.execute_reply.started":"2022-01-08T09:32:58.775931Z","shell.execute_reply":"2022-01-08T09:32:58.815017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"KNN","metadata":{}},{"cell_type":"code","source":"newtrain_descriptors = train_descriptors #train\nnewtest_descriptors = test_descriptors #test","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:33:37.080175Z","iopub.execute_input":"2022-01-08T09:33:37.080726Z","iopub.status.idle":"2022-01-08T09:33:37.083771Z","shell.execute_reply.started":"2022-01-08T09:33:37.080689Z","shell.execute_reply":"2022-01-08T09:33:37.083131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#kfold params\nk_num=[3,5,7] #odd k candidates\nkfoldtrain_acc=[]\nkftrain = KFold(n_splits = 4, random_state = 42, shuffle = True) #4-fold","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:33:56.463589Z","iopub.execute_input":"2022-01-08T09:33:56.463870Z","iopub.status.idle":"2022-01-08T09:33:56.468048Z","shell.execute_reply.started":"2022-01-08T09:33:56.463843Z","shell.execute_reply":"2022-01-08T09:33:56.467439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in k_num:\n  classifier = KNeighborsClassifier(n_neighbors=k)\n  kfacctrain_results = cross_val_score(classifier, newtrain_descriptors, newy_train, cv = kftrain, scoring = 'accuracy')\n  kftrain_accuracy = np.mean(abs(kfacctrain_results))\n  kfoldtrain_acc.append(kftrain_accuracy)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:34:05.720495Z","iopub.execute_input":"2022-01-08T09:34:05.720932Z","iopub.status.idle":"2022-01-08T10:04:57.778306Z","shell.execute_reply.started":"2022-01-08T09:34:05.720890Z","shell.execute_reply":"2022-01-08T10:04:57.777653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#kfold train accuracy\nprint(kfoldtrain_acc)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:04:57.786310Z","iopub.execute_input":"2022-01-08T10:04:57.786579Z","iopub.status.idle":"2022-01-08T10:04:57.804047Z","shell.execute_reply.started":"2022-01-08T10:04:57.786543Z","shell.execute_reply":"2022-01-08T10:04:57.803565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plotting\nplt.plot(k_num, kfoldtrain_acc)\nplt.title('K-Nearest Neighbours using Euclidean Distance')\nplt.xlabel('Number of nearest neighboors')\nplt.ylabel('Accuracy')\nplt.legend(['KFOLD'], loc = 'best')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:04:57.982574Z","iopub.execute_input":"2022-01-08T10:04:57.982848Z","iopub.status.idle":"2022-01-08T10:04:58.143538Z","shell.execute_reply.started":"2022-01-08T10:04:57.982820Z","shell.execute_reply":"2022-01-08T10:04:58.142688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#final classifier\nfinclassifier = KNeighborsClassifier(n_neighbors=5) #k = 5\nfinclassifier = finclassifier.fit(newtrain_descriptors, newy_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:04:58.144644Z","iopub.execute_input":"2022-01-08T10:04:58.145046Z","iopub.status.idle":"2022-01-08T10:04:59.681778Z","shell.execute_reply.started":"2022-01-08T10:04:58.145020Z","shell.execute_reply":"2022-01-08T10:04:59.681099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for test\nfiny_predtest = finclassifier.predict(newtest_descriptors)\ntest_cm = confusion_matrix(newy_test, finy_predtest)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:04:59.682829Z","iopub.execute_input":"2022-01-08T10:04:59.683036Z","iopub.status.idle":"2022-01-08T10:13:19.964946Z","shell.execute_reply.started":"2022-01-08T10:04:59.683011Z","shell.execute_reply":"2022-01-08T10:13:19.964362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for test\nprint(accuracy_score(newy_test, finy_predtest))\nprint(classification_report(newy_test, finy_predtest))\nprint(confusion_matrix(newy_test, finy_predtest))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:19.965806Z","iopub.execute_input":"2022-01-08T10:13:19.966461Z","iopub.status.idle":"2022-01-08T10:13:20.069197Z","shell.execute_reply.started":"2022-01-08T10:13:19.966430Z","shell.execute_reply":"2022-01-08T10:13:20.068663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plotting","metadata":{}},{"cell_type":"code","source":"#heatmap\npandas_cm = pd.DataFrame(test_cm, range(10), range(10)) #0 to 9\nplt.figure(figsize=(10,8))\nsns.set(font_scale=1) #for label size\nsns.heatmap(pandas_cm, annot=True, fmt='g', cmap='RdYlBu') #non-scientific notation\n\n#label\nax = plt.gca() \nax.set_xlabel('Predicted'); ax.set_ylabel('True'); \nax.set_title('Confusion Matrix'); \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:20.070356Z","iopub.execute_input":"2022-01-08T10:13:20.070616Z","iopub.status.idle":"2022-01-08T10:13:20.972527Z","shell.execute_reply.started":"2022-01-08T10:13:20.070582Z","shell.execute_reply":"2022-01-08T10:13:20.971680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#list of correct and error predictions\ncount = 0\nerror = []\ncorrect = []\nfor true, predicted in zip(newy_test, finy_predtest):\n  if true == predicted: \n    correct.append(count) \n  else: \n    error.append(count)\n  count = count + 1","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:20.975464Z","iopub.execute_input":"2022-01-08T10:13:20.975782Z","iopub.status.idle":"2022-01-08T10:13:21.005096Z","shell.execute_reply.started":"2022-01-08T10:13:20.975751Z","shell.execute_reply":"2022-01-08T10:13:21.004441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#correct plotting\nplt.figure(figsize=(24,8))\nfor i, j in enumerate(correct[0:10]):\n  fig = plt.subplot(2, 5, i+1) #2x5 plot\n  fig.imshow(dskewresX_test[j].reshape(28,28), cmap='gray')\n  fig.set_title(\"True: {}, Predicted: {}\" .format(newy_test[j], finy_predtest[j]))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:22.591959Z","iopub.execute_input":"2022-01-08T10:13:22.592414Z","iopub.status.idle":"2022-01-08T10:13:24.196691Z","shell.execute_reply.started":"2022-01-08T10:13:22.592368Z","shell.execute_reply":"2022-01-08T10:13:24.195804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#error plotting\nplt.figure(figsize=(24,8))\nfor k, L in enumerate(error[0:10]):\n  fig = plt.subplot(2, 5, k+1) #2x5 plot\n  fig.imshow(dskewresX_test[L].reshape(28,28), cmap='gray')\n  fig.set_title(\"True: {}, Predicted: {}\" .format(newy_test[L], finy_predtest[L]))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:21.006114Z","iopub.execute_input":"2022-01-08T10:13:21.006632Z","iopub.status.idle":"2022-01-08T10:13:22.590664Z","shell.execute_reply.started":"2022-01-08T10:13:21.006581Z","shell.execute_reply":"2022-01-08T10:13:22.589686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#correct list\nprint(correct)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:24.198083Z","iopub.execute_input":"2022-01-08T10:13:24.198469Z","iopub.status.idle":"2022-01-08T10:13:24.220469Z","shell.execute_reply.started":"2022-01-08T10:13:24.198423Z","shell.execute_reply":"2022-01-08T10:13:24.219652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#error list\nprint(error)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:24.221233Z","iopub.execute_input":"2022-01-08T10:13:24.221428Z","iopub.status.idle":"2022-01-08T10:13:24.225420Z","shell.execute_reply.started":"2022-01-08T10:13:24.221403Z","shell.execute_reply":"2022-01-08T10:13:24.224733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Kaggle submission","metadata":{}},{"cell_type":"code","source":"y_pred_1 = finy_predtest #equate","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:24.226409Z","iopub.execute_input":"2022-01-08T10:13:24.226627Z","iopub.status.idle":"2022-01-08T10:13:24.235948Z","shell.execute_reply.started":"2022-01-08T10:13:24.226583Z","shell.execute_reply":"2022-01-08T10:13:24.235456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_1 #preview","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:24.236973Z","iopub.execute_input":"2022-01-08T10:13:24.237212Z","iopub.status.idle":"2022-01-08T10:13:24.248245Z","shell.execute_reply.started":"2022-01-08T10:13:24.237180Z","shell.execute_reply":"2022-01-08T10:13:24.247355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_pred_1) #check length","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:13:24.249132Z","iopub.execute_input":"2022-01-08T10:13:24.249380Z","iopub.status.idle":"2022-01-08T10:13:24.259419Z","shell.execute_reply.started":"2022-01-08T10:13:24.249344Z","shell.execute_reply":"2022-01-08T10:13:24.258934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Out_df = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:21:04.717529Z","iopub.execute_input":"2022-01-08T10:21:04.718310Z","iopub.status.idle":"2022-01-08T10:21:04.743815Z","shell.execute_reply.started":"2022-01-08T10:21:04.718269Z","shell.execute_reply":"2022-01-08T10:21:04.743205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Out_df['Label'] = y_pred_1","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:21:13.463207Z","iopub.execute_input":"2022-01-08T10:21:13.463917Z","iopub.status.idle":"2022-01-08T10:21:13.468263Z","shell.execute_reply.started":"2022-01-08T10:21:13.463883Z","shell.execute_reply":"2022-01-08T10:21:13.467463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Out_df.to_csv('submission_00.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:21:26.972910Z","iopub.execute_input":"2022-01-08T10:21:26.973191Z","iopub.status.idle":"2022-01-08T10:21:27.027634Z","shell.execute_reply.started":"2022-01-08T10:21:26.973151Z","shell.execute_reply":"2022-01-08T10:21:27.026882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Out_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T10:21:36.572455Z","iopub.execute_input":"2022-01-08T10:21:36.572742Z","iopub.status.idle":"2022-01-08T10:21:36.581112Z","shell.execute_reply.started":"2022-01-08T10:21:36.572703Z","shell.execute_reply":"2022-01-08T10:21:36.580644Z"},"trusted":true},"execution_count":null,"outputs":[]}]}