{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **2301865741 - Edgard Jonathan Putra Pranoto**\n\n# Code for No. 3\n\n**Notes: Explanation is given in the PDF**","metadata":{}},{"cell_type":"code","source":"!pip install opencv-contrib-python==4.4.0.44","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-09T13:46:29.814503Z","iopub.execute_input":"2022-02-09T13:46:29.814809Z","iopub.status.idle":"2022-02-09T13:46:37.508365Z","shell.execute_reply.started":"2022-02-09T13:46:29.814774Z","shell.execute_reply":"2022-02-09T13:46:37.507354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Library","metadata":{}},{"cell_type":"code","source":"import os\nfrom time import time\nimport errno\nimport shutil\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom skimage.feature import hog\nfrom sklearn.datasets import fetch_lfw_people\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.cluster import KMeans\nfrom scipy.cluster.vq import vq\nfrom scipy.spatial.distance import cosine","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:46:37.511100Z","iopub.execute_input":"2022-02-09T13:46:37.511354Z","iopub.status.idle":"2022-02-09T13:46:37.521344Z","shell.execute_reply.started":"2022-02-09T13:46:37.511326Z","shell.execute_reply":"2022-02-09T13:46:37.520729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing dataset\n\n**Extracting Data**","metadata":{}},{"cell_type":"code","source":"import zipfile\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",'r') as z:\n    z.extractall(\".\")","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:46:37.523422Z","iopub.execute_input":"2022-02-09T13:46:37.523755Z","iopub.status.idle":"2022-02-09T13:46:48.759461Z","shell.execute_reply.started":"2022-02-09T13:46:37.523712Z","shell.execute_reply":"2022-02-09T13:46:48.758825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset directory**\n\nSince the dataset is big, i only used 5000 images","metadata":{}},{"cell_type":"code","source":"path = \"./train\"\nfile = os.listdir(path)\n# file = file[:5000]\n\n# jika data tersort command split diatas dan gunakan split yang ada dibawah ini\n_, file = train_test_split(file, test_size=5000, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:46:48.762561Z","iopub.execute_input":"2022-02-09T13:46:48.763219Z","iopub.status.idle":"2022-02-09T13:46:48.795790Z","shell.execute_reply.started":"2022-02-09T13:46:48.763164Z","shell.execute_reply":"2022-02-09T13:46:48.794917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Splitting train and test with 80% for training and 20% for test**","metadata":{}},{"cell_type":"code","source":"train, test = train_test_split(file, test_size=0.2, random_state=42)\nprint(len(train))\nprint(len(test))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:46:48.796988Z","iopub.execute_input":"2022-02-09T13:46:48.797219Z","iopub.status.idle":"2022-02-09T13:46:48.808164Z","shell.execute_reply.started":"2022-02-09T13:46:48.797190Z","shell.execute_reply":"2022-02-09T13:46:48.807537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Read Training Data and Extract the Feature using SIFT**","metadata":{}},{"cell_type":"code","source":"x_train = []\ny_train = []\ntrain_descriptor = []\ntrain_descriptors = []\nIMG_SIZE = 128\n\nSIFT = cv2.SIFT_create()\nfor i,filename in enumerate(tqdm(train)):\n    image = cv2.imread(os.path.join(path, filename), 1)\n    image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n    if(i<5):\n        plt.subplot(1, 5, i+1)\n        plt.imshow(image, cmap='gray')\n        plt.axis('off')\n    \n    kp, ds = SIFT.detectAndCompute(cv2.cvtColor(image,cv2.COLOR_RGB2GRAY),None)\n    if len(kp)>0:\n        x_train.append(image)\n        if filename.split('.')[0] =='cat':\n            y_train.append('cat')\n            train_descriptor.append((\"cat\", ds))\n        else:\n            y_train.append('dog')\n            train_descriptor.append((\"dog\", ds))\n        for d in ds: \n            train_descriptors.append(d)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:46:48.810707Z","iopub.execute_input":"2022-02-09T13:46:48.810933Z","iopub.status.idle":"2022-02-09T13:50:21.636774Z","shell.execute_reply.started":"2022-02-09T13:46:48.810905Z","shell.execute_reply":"2022-02-09T13:50:21.635764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset Frequency for both label in train data**\n\nFrom the frequency the dataset is balanced","metadata":{}},{"cell_type":"code","source":"from collections import Counter\nprint(Counter(y_train))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:50:21.638101Z","iopub.execute_input":"2022-02-09T13:50:21.638902Z","iopub.status.idle":"2022-02-09T13:50:21.644059Z","shell.execute_reply.started":"2022-02-09T13:50:21.638866Z","shell.execute_reply":"2022-02-09T13:50:21.643443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Read Testing Data and Extract the Feature using SIFT**","metadata":{}},{"cell_type":"code","source":"x_test = []\ny_test = []\ntest_descriptor = []\ntest_descriptors = []\n\nfor i,filename in enumerate(tqdm(test)):\n    image = cv2.imread(os.path.join(path, filename), 1)\n    image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n    \n    if(i<5):\n        plt.subplot(1, 5, i+1)\n        plt.imshow(image, cmap='gray')\n        plt.axis('off')\n        \n    kp, ds = SIFT.detectAndCompute(cv2.cvtColor(image, cv2.COLOR_RGB2GRAY ),None)\n    if len(kp)>0:\n        x_test.append(image)\n        if filename.split('.')[0] =='cat':\n            y_test.append('cat')\n            test_descriptor.append((\"cat\", ds))\n        else:\n            y_test.append('dog')\n            test_descriptor.append((\"dog\", ds))\n        for d in ds: \n            test_descriptors.append(d)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:50:21.645206Z","iopub.execute_input":"2022-02-09T13:50:21.645661Z","iopub.status.idle":"2022-02-09T13:51:16.011495Z","shell.execute_reply.started":"2022-02-09T13:50:21.645619Z","shell.execute_reply":"2022-02-09T13:51:16.010571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dataset Frequency for both label in test data**\n\nFrom the frequency the dataset is balanced","metadata":{}},{"cell_type":"code","source":"print(Counter(y_test))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:51:16.013345Z","iopub.execute_input":"2022-02-09T13:51:16.013871Z","iopub.status.idle":"2022-02-09T13:51:16.019649Z","shell.execute_reply.started":"2022-02-09T13:51:16.013828Z","shell.execute_reply":"2022-02-09T13:51:16.018828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**From the dataset, there are more than 2 millions local feature extracted**\n\nSince this will make the computation slow, the extracted feature is then clustered using kmeans to reduce the dimension","metadata":{}},{"cell_type":"code","source":"train_descriptors = np.array(train_descriptors)\ntest_descriptors = np.array(test_descriptors)\nprint(train_descriptors.shape)\nprint(test_descriptors.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:51:16.022785Z","iopub.execute_input":"2022-02-09T13:51:16.023330Z","iopub.status.idle":"2022-02-09T13:51:18.553031Z","shell.execute_reply.started":"2022-02-09T13:51:16.023285Z","shell.execute_reply":"2022-02-09T13:51:18.552034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LABELS = [\"cat\", \"dog\"]\nLABEL2INDEX = {\"cat\": 0, \"dog\": 1}\nINDEX2LABEL = {0: \"cat\", 1: \"dog\"}","metadata":{"execution":{"iopub.status.busy":"2022-02-09T15:37:11.855401Z","iopub.execute_input":"2022-02-09T15:37:11.856237Z","iopub.status.idle":"2022-02-09T15:37:11.861090Z","shell.execute_reply.started":"2022-02-09T15:37:11.856189Z","shell.execute_reply":"2022-02-09T15:37:11.860505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load K Means Model From Question 3","metadata":{}},{"cell_type":"code","source":"import pickle\n\nmodel = pickle.load(open(\"../input/model-comvis-no-3/kmeans (100).sav\", 'rb'))\nk = 100","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:36:25.499006Z","iopub.execute_input":"2022-02-09T14:36:25.499344Z","iopub.status.idle":"2022-02-09T14:36:25.647756Z","shell.execute_reply.started":"2022-02-09T14:36:25.499311Z","shell.execute_reply":"2022-02-09T14:36:25.646845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Frequency diagram for each Visual Word\n\n**Creating Histogram for train local feature**","metadata":{}},{"cell_type":"code","source":"train_features = np.zeros((len(x_train),k),\"float32\")\ny_train = []\nfor i in tqdm(range(len(x_train))):\n    prediction = model.predict(train_descriptor[i][1])\n    y_train.append(train_descriptor[i][0])\n    for w in prediction:\n        train_features[i][w]+=1","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:36:27.209691Z","iopub.execute_input":"2022-02-09T14:36:27.209974Z","iopub.status.idle":"2022-02-09T14:39:08.697365Z","shell.execute_reply.started":"2022-02-09T14:36:27.209945Z","shell.execute_reply":"2022-02-09T14:39:08.696042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating Histogram for test local feature**","metadata":{}},{"cell_type":"code","source":"test_features = np.zeros((len(x_test),k),\"float32\")\ny_test = []\nfor i in tqdm(range(len(x_test))):\n    prediction = model.predict(test_descriptor[i][1])\n    y_test.append(test_descriptor[i][0])\n    for w in prediction:\n        test_features[i][w]+=1","metadata":{"execution":{"iopub.status.busy":"2022-02-09T14:39:08.699060Z","iopub.execute_input":"2022-02-09T14:39:08.699406Z","iopub.status.idle":"2022-02-09T14:39:49.945871Z","shell.execute_reply.started":"2022-02-09T14:39:08.699374Z","shell.execute_reply":"2022-02-09T14:39:49.944827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize the Histogram to [0, 1]","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler().fit(train_features)\ntrain_features = scaler.transform(train_features)\nscaler = StandardScaler().fit(test_features)\ntest_features = scaler.transform(test_features)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T13:54:37.570695Z","iopub.execute_input":"2022-02-09T13:54:37.572436Z","iopub.status.idle":"2022-02-09T13:54:37.591730Z","shell.execute_reply.started":"2022-02-09T13:54:37.572374Z","shell.execute_reply":"2022-02-09T13:54:37.590790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Retrieval using Cosine Similarity\n\n**The document will be ranked and the top-N document will be shown**","metadata":{}},{"cell_type":"code","source":"import random\nfrom scipy.spatial.distance import cosine\n\nfor i in range(0,3):\n    score = []\n    query = random.randint(1,1000)\n    result_size = 10\n    \n    for target in range(len(x_train)):\n        score.append([target,1.0-cosine(test_features[query],train_features[target])])\n    \n    score.sort(key=lambda x: x[1], reverse=True)\n    plt.title(\"Query Image\")\n    plt.imshow(x_test[query],cmap='gray')\n    plt.xticks([])\n    plt.yticks([])\n    plt.show()\n    fig, axs = plt.subplots((result_size+4)//5, 5, figsize=(9,9))\n    for j in range(0,result_size):\n        idx = score[j][0]\n        sim = score[j][1]\n        axs[j//5][j%5].imshow(x_train[idx])\n        axs[j//5][j%5].set_title('Score= %.5lf'%(sim))\n        axs[j//5][j%5].axis('off')\n    plt.subplots_adjust(hspace=-0.5)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T16:48:58.427803Z","iopub.execute_input":"2022-02-09T16:48:58.428106Z","iopub.status.idle":"2022-02-09T16:49:01.577587Z","shell.execute_reply.started":"2022-02-09T16:48:58.428072Z","shell.execute_reply":"2022-02-09T16:49:01.576493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation using Precision and Recall\n\n**The number of document that will be tested is 4**","metadata":{}},{"cell_type":"code","source":"q = 4\nexpected_retrievals = []\nquery_retrievals = []\n\nfor i in range(0,q):\n    print(\"QUERY %d\" % (i+1))\n    expected_retrieval = []\n    query_retrieval = []\n    score = []\n#     query = random.randint(1, 1000)\n    query = len(x_test)-i-1\n    query_label = LABEL2INDEX[y_test[query]]\n    result_size = 25\n\n    for target in range(len(x_train)):\n        score.append([target,1.0-cosine(test_features[query],train_features[target])])\n\n    score.sort(key=lambda x: x[1], reverse=True)\n    plt.title(\"Query Image\")\n    plt.imshow(x_test[query],cmap='gray')\n    plt.xticks([])\n    plt.yticks([])\n    plt.show()\n    for j in range(0,result_size):\n        idx = score[j][0]\n        sim = score[j][1]\n\n        expected_retrieval.append(query_label)\n        retrieval_label = LABEL2INDEX[y_train[idx]]\n        query_retrieval.append(retrieval_label)\n    \n    expected_retrievals.append(expected_retrieval)\n    query_retrievals.append(query_retrieval)\n    print(expected_retrieval)\n    print(query_retrieval)\n    print(\"===============================================\")","metadata":{"execution":{"iopub.status.busy":"2022-02-09T16:47:47.093388Z","iopub.execute_input":"2022-02-09T16:47:47.093953Z","iopub.status.idle":"2022-02-09T16:47:48.526847Z","shell.execute_reply.started":"2022-02-09T16:47:47.093917Z","shell.execute_reply":"2022-02-09T16:47:48.525585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculating Precision and Recall + Plotting ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 4))\nfor i in range(q):\n    expected_result = expected_retrievals[i]\n    query_result = query_retrievals[i]\n    \n    recall = []\n    precision = []\n    true_positive = 0\n    relevant_document = Counter(y_train)[INDEX2LABEL[expected_result[0]]]\n    for j in range(len(query_result)):\n        if query_result[j] == expected_result[j]:\n            true_positive += 1\n        recall.append((true_positive/relevant_document))\n        precision.append((true_positive/(j+1)))\n    interpolated_precision = np.maximum.accumulate(precision[::-1])[::-1]\n    \n    plt.subplot(1, q, i+1)\n    plt.title(\"Query %d (Expected result: %s)\"%(i+1, INDEX2LABEL[expected_result[0]]))\n    plt.step(recall, interpolated_precision, '-r', label=\"interpolated\")\n    plt.xlabel(\"Recall\")\n    plt.ylabel(\"Precision\")\n    plt.xlim(0.0)\n    plt.ylim((0.0, 1.1))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T16:47:52.060829Z","iopub.execute_input":"2022-02-09T16:47:52.061108Z","iopub.status.idle":"2022-02-09T16:47:52.511185Z","shell.execute_reply.started":"2022-02-09T16:47:52.061078Z","shell.execute_reply":"2022-02-09T16:47:52.510320Z"},"trusted":true},"execution_count":null,"outputs":[]}]}