{"cells":[{"metadata":{"_uuid":"18302521565d04886a60e4479509fc505db914dc"},"cell_type":"markdown","source":"### Import Libraries and read data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport scipy.io\nimport warnings\nimport os\nprint(os.listdir(\"../input\"))\nwarnings.filterwarnings('ignore')\n\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(\"Shape of Training data = \", train.shape)\nprint(\"Sahpe of Test Data = \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ebde4c5b4f5b462d85ef3813cc8c23e3e49676e"},"cell_type":"markdown","source":"#### Separate Features and Prediction"},{"metadata":{"trusted":true,"_uuid":"cd9ad9d5aeae5bcf322e10896d53e10ee982aec8"},"cell_type":"code","source":"PC = train.iloc[:, 0:1]\ntrain = train.iloc[:, 1:]\nprint(\"trainX = \", train.shape, \"trainY = \", PC.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f815c1e477a61ab88333fce6c39e3bc8bf196f60"},"cell_type":"markdown","source":"### Remove Columns with Less Variance"},{"metadata":{"trusted":true,"_uuid":"f0cd4345521c45d21c947a5078b8ac38cabf1446"},"cell_type":"code","source":"variance = np.var(train, axis = 0)>1000\ntrain = train.loc[:, variance]\ntest = test.loc[:, variance]\nprint(\"Shape of Training data = \", train.shape)\nprint(\"Sahpe of Test Data = \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc43e5979ce0561862de357ac5eb623dcac1fd94"},"cell_type":"markdown","source":"#### Split Data in Training and Validation Set\n- Training Data: 80%\n- Validation Data: 20%"},{"metadata":{"trusted":true,"_uuid":"bcfd91079e43220ae79fe9c6f6d7754e858be0ea"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_X, val_X, train_Y, val_Y = train_test_split(train, PC, test_size = 0.2, \n                                                 random_state = 0)\nprint(\"train_X Shape = \", train_X.shape, \"train_Y Shape = \", train_Y.shape)\nprint(\"val_X Shape = \", val_X.shape, \"val_Y Shape = \", val_Y.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e7583e9a6698902b63f41cef96e53270122267e"},"cell_type":"markdown","source":"### Dimensionality Reduction Using PCA\n> We will choose top n components which can represent 80% variance of data"},{"metadata":{"trusted":true,"_uuid":"ea1510d2650bcd11609ae1298db3fc41b20b1d56"},"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=0.8)\npca.fit(train_X)\ntrain_X = pca.transform(train_X)\nval_X = pca.transform(val_X)\ntest = pca.transform(test)\nprint(\"train_X Shape = \", train_X.shape)\nprint(\"val_X Shape = \", val_X.shape)\nprint(\"test Shape = \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"493fc1886dae3cb1263d6b2dcefeccadb670cf38"},"cell_type":"markdown","source":"### Normalize Data"},{"metadata":{"trusted":true,"_uuid":"4ed4f2c0fa013f440699a73a258e227d4336a127"},"cell_type":"code","source":"def normalize(sigma, mean, X):\n    X = (X - mean)/sigma\n    return X\n    \nsigma = np.std(train_X, axis = 0)\nmean = np.mean(train_X, axis = 0)\n\ntrain_X = normalize(sigma, mean, train_X)\nval_X = normalize(sigma, mean, val_X)\ntest = normalize(sigma, mean, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f1b1d875636910fa3061c4d60674375e53a824f"},"cell_type":"code","source":"print(train_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac156b55412eb7ff622f6e83f375a90e37b2af2f"},"cell_type":"code","source":"train.isna().any().any()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15c0a0a38e36cfad898e130d0435e3eec53ce8f8"},"cell_type":"markdown","source":"### Custome k nearest neighbour classifier"},{"metadata":{"trusted":true,"_uuid":"4ea239d779c6706487abc4dcade5c7d10ceca8fb"},"cell_type":"code","source":"l1Dist = np.array([[3],\n                  [1],\n                  [2]])\ny = np.array([[1],\n             [2],\n             [3],\n             [4]])\nind = np.argsort(l1Dist, axis = 0)\nprint(ind)\nprint(y[ind[:2,0]].ravel())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5139b4f718034c4c3a8e2873e0f61f3bdbc3ae4"},"cell_type":"code","source":"class KNN:\n    def __init__(self, k = 5):\n        self.k = k\n    #X.shape = [1, n]\n    def l1(self, X):\n        return np.sum(np.abs(X - self.X_train), axis = 1)\n    #X.shape = [number of examples, feature] = [m, n]\n    #y.shape = [number of examples, 1]\n    def fit(self, X, y):\n        self.X_train = X\n        self.Y_train = y.reshape(y.shape[0], 1)\n        \n    def predictHelper(self, l1Dist):\n        l1Dist = l1Dist.reshape(l1Dist.shape[0], 1)\n        l1Arg = np.argsort(l1Dist, axis = 0)[:self.k,0]\n        counts = np.bincount(self.Y_train[l1Arg].ravel())\n        return np.argmax(counts)\n    \n    def predict(self, X_test):\n        y_test = np.zeros((X_test.shape[0], 1))\n        for i in range(X_test.shape[0]):\n            l1Dist = self.l1(X_test[i:i+1])\n            y_test[i] = self.predictHelper(l1Dist)\n        \n        return y_test.ravel().astype(np.int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4aff56a1956070c3d6239faba28d009f751ac1e"},"cell_type":"code","source":"model = KNN(5)\nmodel.fit(train_X, train_Y.values)\ny_test = model.predict(val_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cecee0b2f04e78d8a9f95bba7e2e2d2ac2fb8d6c"},"cell_type":"code","source":"boolMap =  (y_test == val_Y.iloc[:,0].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be3d996a2125267526ad714e1f5006ecb86d6a0f"},"cell_type":"code","source":"acc = np.sum(boolMap)/y_test.shape[0]\nprint(acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bafac602a0f63973e62cc295631702d1c2d1992"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['ImageId'] = np.arange(1, test.shape[0]+1)\nsub['Label'] = model.predict(test)\nsub.to_csv(\"outKNN.csv\", index = False, header = True)\nsub.head(n = 10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88028761ec56276f7c7b98b3b348af2853e0181c"},"cell_type":"markdown","source":"### Train using sklearn KNN Classifier"},{"metadata":{"trusted":true,"_uuid":"0922f9581916ef3af72f5f4cdb0bfa7580296f0a"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.neighbors import KNeighborsClassifier\nneigh = KNeighborsClassifier(n_neighbors=10)\nneigh.fit(train_X, train_Y)\nprint(\"Training Error = \", neigh.score(train_X, train_Y))\nprint(\"Validation Error = \", neigh.score(val_X, val_Y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8876655890594c6609e1148fe0f91293666b9c85"},"cell_type":"code","source":"(val_Y.values>0).sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91af1c6f47f061a49a18c79924a5eab58c2b51ce"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['ImageId'] = np.arange(1, test.shape[0]+1)\nsub['Label'] = neigh.predict(test)\nsub.to_csv(\"outKNN1.csv\", index = False, header = True)\nsub.head(n = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afc4c2f7c6ed638feaa1dd3b0b8fdeb302d56954"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc85d075858e7de211ef0f26c0f7f0bd9ad17491"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97c93a2fc39dd061b605948f7f0023f56d313431"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}