{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **2301865741 - Edgard Jonathan Putra Pranoto**\n\n# Code for No. 3\n\n**Notes: Explanation is given in the PDF**","metadata":{}},{"cell_type":"code","source":"!pip install opencv-contrib-python==4.4.0.44","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:22:24.657089Z","iopub.execute_input":"2022-02-09T14:22:24.657332Z","iopub.status.idle":"2022-02-09T14:22:35.974600Z","shell.execute_reply.started":"2022-02-09T14:22:24.657264Z","shell.execute_reply":"2022-02-09T14:22:35.974130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Library","metadata":{}},{"cell_type":"code","source":"import os\nfrom time import time\nimport errno\nimport shutil\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom skimage.feature import hog\nfrom sklearn.datasets import fetch_lfw_people\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.cluster import KMeans\nfrom scipy.cluster.vq import vq\nfrom scipy.spatial.distance import cosine","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-09T14:22:35.975872Z","iopub.execute_input":"2022-02-09T14:22:35.976042Z","iopub.status.idle":"2022-02-09T14:22:37.562805Z","shell.execute_reply.started":"2022-02-09T14:22:35.976019Z","shell.execute_reply":"2022-02-09T14:22:37.562003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing dataset\n\n**Extracting Data**","metadata":{}},{"cell_type":"code","source":"import zipfile\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",'r') as z:\n    z.extractall(\".\")","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:22:37.563578Z","iopub.execute_input":"2022-02-09T14:22:37.563727Z","iopub.status.idle":"2022-02-09T14:22:49.886330Z","shell.execute_reply.started":"2022-02-09T14:22:37.563707Z","shell.execute_reply":"2022-02-09T14:22:49.884933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset directory**\n\nSince the dataset is big, i only used 5000 images","metadata":{}},{"cell_type":"code","source":"path = \"./train\"\nfile = os.listdir(path)\nfile = file[:5000]\n\n# jika data tersort command split diatas dan gunakan split yang ada dibawah ini\n# _, file = train_test_split(file, test_size=5000, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:22:49.888036Z","iopub.execute_input":"2022-02-09T14:22:49.888258Z","iopub.status.idle":"2022-02-09T14:22:49.905962Z","shell.execute_reply.started":"2022-02-09T14:22:49.888236Z","shell.execute_reply":"2022-02-09T14:22:49.904727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Splitting train and test with 80% for training and 20% for test**","metadata":{}},{"cell_type":"code","source":"train, test = train_test_split(file, test_size=0.2, random_state=42)\nprint(\"Train Data Count: %d\" % len(train))\nprint(\"Test Data Count: %d\" % len(test))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:22:49.907441Z","iopub.execute_input":"2022-02-09T14:22:49.908202Z","iopub.status.idle":"2022-02-09T14:22:50.130265Z","shell.execute_reply.started":"2022-02-09T14:22:49.908166Z","shell.execute_reply":"2022-02-09T14:22:50.129369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Read Training Data and Extract the Feature using SIFT**","metadata":{}},{"cell_type":"code","source":"x_train = []\ny_train = []\ntrain_descriptor = []\ntrain_descriptors = []\nIMG_SIZE = 128\n\nSIFT = cv2.SIFT_create()\nfor i,filename in enumerate(tqdm(train)):\n    image = cv2.imread(os.path.join(path, filename), 0)\n    \n    if(i<5):\n        plt.subplot(1, 5, i+1)\n        plt.imshow(image, cmap='gray')\n        plt.axis('off')\n    \n    kp, ds = SIFT.detectAndCompute(image,None)\n    if len(kp)>0:\n        x_train.append(image)\n        if filename.split('.')[0] =='cat':\n            y_train.append('cat')\n            train_descriptor.append((\"cat\", ds))\n        else:\n            y_train.append('dog')\n            train_descriptor.append((\"dog\", ds))\n        for d in ds: \n            train_descriptors.append(d)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:22:50.131318Z","iopub.execute_input":"2022-02-09T14:22:50.131491Z","iopub.status.idle":"2022-02-09T14:25:41.685395Z","shell.execute_reply.started":"2022-02-09T14:22:50.131470Z","shell.execute_reply":"2022-02-09T14:25:41.684105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset Frequency for both label in train data**\n\nFrom the frequency the dataset is balanced","metadata":{}},{"cell_type":"code","source":"from collections import Counter\nprint(Counter(y_train))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:25:41.686760Z","iopub.execute_input":"2022-02-09T14:25:41.686966Z","iopub.status.idle":"2022-02-09T14:25:41.692159Z","shell.execute_reply.started":"2022-02-09T14:25:41.686943Z","shell.execute_reply":"2022-02-09T14:25:41.691551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Read Testing Data and Extract the Feature using SIFT**","metadata":{}},{"cell_type":"code","source":"x_test = []\ny_test = []\ntest_descriptor = []\ntest_descriptors = []\n\nfor i,filename in enumerate(tqdm(test)):\n    image = cv2.imread(os.path.join(path, filename), 0)\n    \n    if(i<5):\n        plt.subplot(1, 5, i+1)\n        plt.imshow(image, cmap='gray')\n        plt.axis('off')\n        \n    kp, ds = SIFT.detectAndCompute(image,None)\n    if len(kp)>0:\n        x_test.append(image)\n        if filename.split('.')[0] =='cat':\n            y_test.append('cat')\n            test_descriptor.append((\"cat\", ds))\n        else:\n            y_test.append('dog')\n            test_descriptor.append((\"dog\", ds))\n        for d in ds: \n            test_descriptors.append(d)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:25:41.692962Z","iopub.execute_input":"2022-02-09T14:25:41.693138Z","iopub.status.idle":"2022-02-09T14:26:23.028256Z","shell.execute_reply.started":"2022-02-09T14:25:41.693117Z","shell.execute_reply":"2022-02-09T14:26:23.025933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset Frequency for both label in test data**\n\nFrom the frequency the dataset is balanced","metadata":{}},{"cell_type":"code","source":"print(Counter(y_test))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:26:23.029322Z","iopub.execute_input":"2022-02-09T14:26:23.029482Z","iopub.status.idle":"2022-02-09T14:26:23.034511Z","shell.execute_reply.started":"2022-02-09T14:26:23.029461Z","shell.execute_reply":"2022-02-09T14:26:23.033607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**From the dataset, there are more than 2 millions local feature extracted**\n\nSince this will make the computation slow, the extracted feature is then clustered using kmeans to reduce the dimension","metadata":{}},{"cell_type":"code","source":"train_descriptors = np.array(train_descriptors)\ntest_descriptors = np.array(test_descriptors)\nprint(\"Count of Local Feature: %d\" % train_descriptors.shape[0])\nprint(\"Count of Local Feature: %d\" % test_descriptors.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:26:23.037416Z","iopub.execute_input":"2022-02-09T14:26:23.037589Z","iopub.status.idle":"2022-02-09T14:26:25.177740Z","shell.execute_reply.started":"2022-02-09T14:26:23.037569Z","shell.execute_reply":"2022-02-09T14:26:25.176814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Elbow method for finding the best number of cluster\n\nSince the clustering time is long, i skipped some number and only checked 5 number of cluster","metadata":{}},{"cell_type":"code","source":"# all_k = [1,25,50, 75, 100]\n# error = []\n# for k in all_k:\n#     model = KMeans(k)\n#     model.fit(train_descriptors)\n#     error.append(model.inertia_)\n\n# plt.plot(all_k, error)\n# plt.xlabel('K')\n# plt.ylabel('Inertia')\n# plt.title('The Elbow Method')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:26:25.182554Z","iopub.execute_input":"2022-02-09T14:26:25.182711Z","iopub.status.idle":"2022-02-09T14:26:25.185411Z","shell.execute_reply.started":"2022-02-09T14:26:25.182691Z","shell.execute_reply":"2022-02-09T14:26:25.184995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering using Kmeans (Visual Words)\n\nthe number cluster is from elbow method that is done above which is 50","metadata":{}},{"cell_type":"code","source":"import math\n\nk = 50\n\nmodel = KMeans(n_clusters = k, n_init = 1, max_iter = 20, verbose = 1)\nmodel.fit(train_descriptors)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:26:25.186097Z","iopub.execute_input":"2022-02-09T14:26:25.186229Z","iopub.status.idle":"2022-02-09T14:30:06.662720Z","shell.execute_reply.started":"2022-02-09T14:26:25.186210Z","shell.execute_reply":"2022-02-09T14:30:06.662219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Frequency diagram for each Visual Word","metadata":{}},{"cell_type":"markdown","source":"**Creating Histogram for train local feature**","metadata":{}},{"cell_type":"code","source":"train_features = np.zeros((len(x_train),k),\"float32\")\ny_train = []\nfor i in tqdm(range(len(x_train))):\n    prediction = model.predict(train_descriptor[i][1])\n    y_train.append(train_descriptor[i][0])\n    for w in prediction:\n        train_features[i][w]+=1","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:30:06.665571Z","iopub.execute_input":"2022-02-09T14:30:06.665897Z","iopub.status.idle":"2022-02-09T14:32:23.370610Z","shell.execute_reply.started":"2022-02-09T14:30:06.665863Z","shell.execute_reply":"2022-02-09T14:32:23.370093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating Histogram for test local feature**","metadata":{}},{"cell_type":"code","source":"test_features = np.zeros((len(x_test),k),\"float32\")\ny_test = []\nfor i in tqdm(range(len(x_test))):\n    prediction = model.predict(test_descriptor[i][1])\n    y_test.append(test_descriptor[i][0])\n    for w in prediction:\n        test_features[i][w]+=1","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:32:23.373652Z","iopub.execute_input":"2022-02-09T14:32:23.375356Z","iopub.status.idle":"2022-02-09T14:32:57.199433Z","shell.execute_reply.started":"2022-02-09T14:32:23.375317Z","shell.execute_reply":"2022-02-09T14:32:57.198916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LABELS = [\"cat\", \"dog\"]\nLABEL2INDEX = {\"cat\": 0, \"dog\": 1}\nINDEX2LABEL = {0: \"cat\", 1: \"dog\"}","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:32:57.202529Z","iopub.execute_input":"2022-02-09T14:32:57.202916Z","iopub.status.idle":"2022-02-09T14:32:57.207801Z","shell.execute_reply.started":"2022-02-09T14:32:57.202885Z","shell.execute_reply":"2022-02-09T14:32:57.207085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_train = [LABEL2INDEX[label] for label in y_train]\n# y_test = [LABEL2INDEX[label] for label in y_test]\n\n# y_train = [INDEX2LABEL[label] for label in y_train]\n# y_test = [INDEX2LABEL[label] for label in y_test]","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:32:57.209160Z","iopub.execute_input":"2022-02-09T14:32:57.210634Z","iopub.status.idle":"2022-02-09T14:32:57.223413Z","shell.execute_reply.started":"2022-02-09T14:32:57.210596Z","shell.execute_reply":"2022-02-09T14:32:57.222570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finding the best number of neighbors for KNN","metadata":{}},{"cell_type":"code","source":"import math\n\nn_neighbors = 3\nbest_acc = 0\nbest_neighbors = 0\nneighbors = []\nacc = []\n\nfor z in range(1,10):\n    KNN = KNeighborsClassifier(n_neighbors = n_neighbors)\n    KNN.fit(train_features, y_train)\n    \n    y_pred = KNN.predict(test_features)\n\n    # y_test = [INDEX2LABEL[index] for index in y_test]\n    # y_pred = [INDEX2LABEL[index] for index in y_pred]\n\n    accuracy = (accuracy_score(y_test, y_pred))\n    acc.append(accuracy)\n    neighbors.append(n_neighbors)\n    \n    if(best_acc<accuracy):\n        best_acc=accuracy\n        best_neighbors = n_neighbors\n    \n    n_neighbors+=14","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:32:57.224711Z","iopub.execute_input":"2022-02-09T14:32:57.227837Z","iopub.status.idle":"2022-02-09T14:33:04.142354Z","shell.execute_reply.started":"2022-02-09T14:32:57.227785Z","shell.execute_reply":"2022-02-09T14:33:04.141604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Plot for the classifier**","metadata":{}},{"cell_type":"code","source":"plt.plot(neighbors, acc)\nplt.xlabel('n_neighbors')\nplt.ylabel('accuracy')\nplt.title('accuracy based on neighbors')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:33:04.143376Z","iopub.execute_input":"2022-02-09T14:33:04.143558Z","iopub.status.idle":"2022-02-09T14:33:04.294824Z","shell.execute_reply.started":"2022-02-09T14:33:04.143533Z","shell.execute_reply":"2022-02-09T14:33:04.294035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the Classifier based on the best number of neighbors","metadata":{}},{"cell_type":"code","source":"KNN = KNeighborsClassifier(n_neighbors = best_neighbors)\nKNN.fit(train_features, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:33:04.296107Z","iopub.execute_input":"2022-02-09T14:33:04.296337Z","iopub.status.idle":"2022-02-09T14:33:04.418680Z","shell.execute_reply.started":"2022-02-09T14:33:04.296305Z","shell.execute_reply":"2022-02-09T14:33:04.417925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating the Classifier using Classification Report and Confusion Matrix","metadata":{}},{"cell_type":"code","source":"y_pred = KNN.predict(test_features)\n\n# y_test = [INDEX2LABEL[index] for index in y_test]\n# y_pred = [INDEX2LABEL[index] for index in y_pred]\n\naccuracy = (accuracy_score(y_test, y_pred))\n\nprint(\"Accuracy = %.2lf\\n\" % (accuracy*100))\n\nreport = classification_report(y_test, y_pred)\nprint(report)\n\ncf_matrix = confusion_matrix(y_test, y_pred)\ncf_matrix = pd.DataFrame(cf_matrix, index = [\"cat\", \"dog\"],\n                  columns = [\"cat\", \"dog\"])\nplt.figure(figsize = (10,7))\nsns.heatmap(cf_matrix, annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:33:04.420504Z","iopub.execute_input":"2022-02-09T14:33:04.421147Z","iopub.status.idle":"2022-02-09T14:33:05.736588Z","shell.execute_reply.started":"2022-02-09T14:33:04.421113Z","shell.execute_reply":"2022-02-09T14:33:05.735811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving the Model","metadata":{}},{"cell_type":"code","source":"import pickle\n\nfilename = 'kmeans.sav'\npickle.dump(model, open(filename, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:33:05.737659Z","iopub.execute_input":"2022-02-09T14:33:05.737921Z","iopub.status.idle":"2022-02-09T14:33:05.751743Z","shell.execute_reply.started":"2022-02-09T14:33:05.737899Z","shell.execute_reply":"2022-02-09T14:33:05.751255Z"},"trusted":true},"execution_count":null,"outputs":[]}]}