{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70497,"databundleVersionId":7713244,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":2507.024856,"end_time":"2024-04-23T03:52:06.826484","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-04-23T03:10:19.801628","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import dependencies ","metadata":{"papermill":{"duration":0.008238,"end_time":"2024-04-23T03:10:22.940759","exception":false,"start_time":"2024-04-23T03:10:22.932521","status":"completed"},"tags":[]}},{"cell_type":"code","source":" \n!pip install scikit-learn-extra\n","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:10:22.957137Z","iopub.status.busy":"2024-04-23T03:10:22.956770Z","iopub.status.idle":"2024-04-23T03:10:38.445865Z","shell.execute_reply":"2024-04-23T03:10:38.444663Z"},"papermill":{"duration":15.500257,"end_time":"2024-04-23T03:10:38.448413","exception":false,"start_time":"2024-04-23T03:10:22.948156","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport os\n\n#for mathematics(most use the array since we did the classification)\nimport numpy as np\n#for explore data analysis\nimport pandas as pd\nfrom PIL import Image\nfrom pathlib import Path\nfrom pathlib import Path\nfrom fastai.vision.all import *\nfrom fastai.callback import *\nfrom fastai.metrics import error_rate\nfrom fastai.callback.tracker import EarlyStoppingCallback\nfrom fastai.vision.all import get_image_files\n\n#for image processing\nimport matplotlib.image as mpimg\n\n#tersorflow for data genertor \nfrom tensorflow import keras\nfrom zipfile import ZipFile\nfrom os import listdir, walk\nfrom os.path import isfile, join\nfrom tensorflow.keras import Input, Model\nfrom tensorflow.keras.applications import ResNet101V2\nfrom tensorflow.keras.applications.imagenet_utils import preprocess_input\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense\nfrom tensorflow.keras.optimizers import Adam\n\n#Kmean and k-menoid for clustering classification\nfrom sklearn.cluster import KMeans\nfrom sklearn_extra.cluster import KMedoids\nfrom sklearn.metrics import silhouette_score\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-04-23T03:10:38.466972Z","iopub.status.busy":"2024-04-23T03:10:38.466605Z","iopub.status.idle":"2024-04-23T03:11:01.383053Z","shell.execute_reply":"2024-04-23T03:11:01.381367Z"},"papermill":{"duration":22.929231,"end_time":"2024-04-23T03:11:01.386043","exception":false,"start_time":"2024-04-23T03:10:38.456812","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 224\n\ndatagen = ImageDataGenerator(\n        samplewise_center=True,\n        rescale=1/255.)\n\ntrain_it = datagen.flow_from_directory('../input/csc532-fruits-categorical-classification-2024/csc532_2024_train/train', \n                                       target_size=(224, 224), \n                                       color_mode='rgb', \n                                       class_mode=\"categorical\")\n\nvalid_it = datagen.flow_from_directory('../input/csc532-fruits-categorical-classification-2024/csc532_2024_test/test', \n                                       target_size=(224, 224), \n                                       color_mode='rgb', \n                                       class_mode=\"categorical\")","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:11:01.405280Z","iopub.status.busy":"2024-04-23T03:11:01.404667Z","iopub.status.idle":"2024-04-23T03:11:11.723024Z","shell.execute_reply":"2024-04-23T03:11:11.722203Z"},"papermill":{"duration":10.3303,"end_time":"2024-04-23T03:11:11.725240","exception":false,"start_time":"2024-04-23T03:11:01.394940","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{"papermill":{"duration":0.008698,"end_time":"2024-04-23T03:11:11.742957","exception":false,"start_time":"2024-04-23T03:11:11.734259","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#check length\nprint(\"length for training dataset\",len(train_it))\nprint(\"length for testing dataset\",len(valid_it))","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:11:11.762443Z","iopub.status.busy":"2024-04-23T03:11:11.762078Z","iopub.status.idle":"2024-04-23T03:11:11.767300Z","shell.execute_reply":"2024-04-23T03:11:11.766354Z"},"papermill":{"duration":0.01862,"end_time":"2024-04-23T03:11:11.769822","exception":false,"start_time":"2024-04-23T03:11:11.751202","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load the image to display images in train dataset\ntrain_it_image = ImageDataLoaders.from_folder(path = \"/kaggle/input/csc532-fruits-categorical-classification-2024/csc532_2024_train/train\", train='train', valid_pct=0.2, \n                                          item_tfms=Resize(460), \n                                          batch_tfms=[*aug_transforms(size=224), Normalize.from_stats(*imagenet_stats)])","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:11:11.788002Z","iopub.status.busy":"2024-04-23T03:11:11.787713Z","iopub.status.idle":"2024-04-23T03:11:15.239463Z","shell.execute_reply":"2024-04-23T03:11:15.238606Z"},"papermill":{"duration":3.463731,"end_time":"2024-04-23T03:11:15.242027","exception":false,"start_time":"2024-04-23T03:11:11.778296","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show image with their label\ntrain_it_image.show_batch()","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:11:15.262242Z","iopub.status.busy":"2024-04-23T03:11:15.261416Z","iopub.status.idle":"2024-04-23T03:11:17.896258Z","shell.execute_reply":"2024-04-23T03:11:17.895239Z"},"papermill":{"duration":2.653544,"end_time":"2024-04-23T03:11:17.904778","exception":false,"start_time":"2024-04-23T03:11:15.251234","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load ImageNet Base Model\n","metadata":{"execution":{"iopub.execute_input":"2024-03-26T15:20:21.175247Z","iopub.status.busy":"2024-03-26T15:20:21.174719Z","iopub.status.idle":"2024-03-26T15:20:26.84099Z","shell.execute_reply":"2024-03-26T15:20:26.839957Z","shell.execute_reply.started":"2024-03-26T15:20:21.175206Z"},"papermill":{"duration":0.0165,"end_time":"2024-04-23T03:11:17.938099","exception":false,"start_time":"2024-04-23T03:11:17.921599","status":"completed"},"tags":[]}},{"cell_type":"code","source":"base_model = keras.applications.InceptionV3(\n    weights='imagenet',\n    input_shape=(224, 224, 3),\n    include_top=False)\n\n# Freeze base model\nbase_model.trainable = False\n\n#We freeze base model for preventing overfitting, maintaining stability, and preserving valuable learned knowledge.\n","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:11:17.973005Z","iopub.status.busy":"2024-04-23T03:11:17.972297Z","iopub.status.idle":"2024-04-23T03:11:20.843556Z","shell.execute_reply":"2024-04-23T03:11:20.842426Z"},"papermill":{"duration":2.891911,"end_time":"2024-04-23T03:11:20.846240","exception":false,"start_time":"2024-04-23T03:11:17.954329","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract feature to base_model","metadata":{"papermill":{"duration":0.018066,"end_time":"2024-04-23T03:11:20.882922","exception":false,"start_time":"2024-04-23T03:11:20.864856","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_features = base_model.predict(train_it)\nvalid_features = base_model.predict(valid_it)","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:11:20.920796Z","iopub.status.busy":"2024-04-23T03:11:20.920421Z","iopub.status.idle":"2024-04-23T03:14:52.148429Z","shell.execute_reply":"2024-04-23T03:14:52.147169Z"},"papermill":{"duration":211.25014,"end_time":"2024-04-23T03:14:52.151135","exception":false,"start_time":"2024-04-23T03:11:20.900995","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Flatten the features\n","metadata":{"papermill":{"duration":0.056434,"end_time":"2024-04-23T03:14:52.263049","exception":false,"start_time":"2024-04-23T03:14:52.206615","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#change the dimension on every data into one dimension with .reshape[0][-1]\ntrain_features_flat = train_features.reshape(train_features.shape[0], -1)\nvalid_features_flat = valid_features.reshape(valid_features.shape[0], -1)","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:14:52.375201Z","iopub.status.busy":"2024-04-23T03:14:52.374473Z","iopub.status.idle":"2024-04-23T03:14:52.379713Z","shell.execute_reply":"2024-04-23T03:14:52.378764Z"},"papermill":{"duration":0.062937,"end_time":"2024-04-23T03:14:52.381887","exception":false,"start_time":"2024-04-23T03:14:52.318950","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add dense layer","metadata":{"papermill":{"duration":0.05393,"end_time":"2024-04-23T03:14:52.488857","exception":false,"start_time":"2024-04-23T03:14:52.434927","status":"completed"},"tags":[]}},{"cell_type":"code","source":"inputs = Input(shape=(224, 224, 3))\n\nx = base_model(inputs, training=False)\nx = GlobalAveragePooling2D()(x)\n\noutputs = Dense(len(train_it.class_indices), activation = 'softmax')(x)\n\nmodel = Model(inputs, outputs)","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:14:52.598856Z","iopub.status.busy":"2024-04-23T03:14:52.598025Z","iopub.status.idle":"2024-04-23T03:14:52.622459Z","shell.execute_reply":"2024-04-23T03:14:52.621715Z"},"papermill":{"duration":0.081724,"end_time":"2024-04-23T03:14:52.624660","exception":false,"start_time":"2024-04-23T03:14:52.542936","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add Layers to Model\n","metadata":{"papermill":{"duration":0.053847,"end_time":"2024-04-23T03:14:52.733325","exception":false,"start_time":"2024-04-23T03:14:52.679478","status":"completed"},"tags":[]}},{"cell_type":"code","source":"inputs = Input(shape=(224, 224, 3))\n\nx = base_model(inputs, training=False)\nx = GlobalAveragePooling2D()(x)\n\noutputs = Dense(len(train_it.class_indices), activation = 'softmax')(x)\n\nmodel = Model(inputs, outputs)","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:14:52.892350Z","iopub.status.busy":"2024-04-23T03:14:52.891961Z","iopub.status.idle":"2024-04-23T03:14:52.913900Z","shell.execute_reply":"2024-04-23T03:14:52.912918Z"},"papermill":{"duration":0.127936,"end_time":"2024-04-23T03:14:52.916193","exception":false,"start_time":"2024-04-23T03:14:52.788257","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Right now we will get the model to predict, but the goals is applying with kmean and kmenoids","metadata":{"papermill":{"duration":0.054433,"end_time":"2024-04-23T03:14:53.025916","exception":false,"start_time":"2024-04-23T03:14:52.971483","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Kmean and Kmediods \nwe will go dive deep into clustering part, but before we get hands on it.\nwhat is silhouette_score?\n- The silhouette score is a metric used to evaluate the quality of clusters created by clustering algorithms, such as KMeans. It provides a measure of how similar an object is to its own cluster compared to other clusters. The silhouette score ranges from -1 to 1, where:\n\n - A score close to +1 indicates that the sample is far away from the neighboring clusters.\n - A score of 0 indicates that the sample is on or very close to the decision boundary   between two neighboring clusters.\n - A score close to -1 indicates that the sample may have been assigned to the wrong cluster.","metadata":{"papermill":{"duration":0.05274,"end_time":"2024-04-23T03:14:53.132413","exception":false,"start_time":"2024-04-23T03:14:53.079673","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Clustering with kmean ","metadata":{"papermill":{"duration":0.054239,"end_time":"2024-04-23T03:14:53.241314","exception":false,"start_time":"2024-04-23T03:14:53.187075","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"we can see the score of each number k from log from this following box","metadata":{"papermill":{"duration":0.056636,"end_time":"2024-04-23T03:14:53.355948","exception":false,"start_time":"2024-04-23T03:14:53.299312","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import math\nfrom sklearn.metrics import silhouette_score\nfrom sklearn.cluster import KMeans\n\n# Cluster with k=2, 3, 6\nk_values = [2, 3, 6]\n\nfor k in k_values:\n    kmeans = KMeans(n_clusters=k, random_state=42)\n    train_cluster_labels = kmeans.fit_predict(train_features_flat)\n    valid_cluster_labels = kmeans.predict(valid_features_flat)\n\n    # Assessing the validity of clustered outcome (Silhouette Score)\n    train_silhouette_score = silhouette_score(train_features_flat, train_cluster_labels)\n    valid_silhouette_score = silhouette_score(valid_features_flat, valid_cluster_labels)\n    print(f\"KMeans with k={k}: Train Silhouette Score: {train_silhouette_score}, Valid Silhouette Score: {valid_silhouette_score}\")\n\n    # Model Fine Tuning\n    base_model.trainable = True\n    fine_tune_at = 100\n\n    for layer in base_model.layers[:fine_tune_at]:\n        layer.trainable = False\n\n    model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001 / 10), metrics=[\"accuracy\"])\n\n    # Calculate steps_per_epoch and validation_steps as integers\n    steps_per_epoch = math.ceil(train_it.samples / train_it.batch_size)\n    validation_steps = math.ceil(valid_it.samples / valid_it.batch_size)\n\n    # Model fitting\n    history = model.fit(train_it,\n                        validation_data=valid_it,\n                        steps_per_epoch=steps_per_epoch,\n                        validation_steps=validation_steps,\n                        epochs=5)\n","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:14:53.470034Z","iopub.status.busy":"2024-04-23T03:14:53.469636Z","iopub.status.idle":"2024-04-23T03:51:18.557700Z","shell.execute_reply":"2024-04-23T03:51:18.556757Z"},"papermill":{"duration":2185.147886,"end_time":"2024-04-23T03:51:18.560034","exception":false,"start_time":"2024-04-23T03:14:53.412148","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering with kmediods ","metadata":{"papermill":{"duration":0.328749,"end_time":"2024-04-23T03:51:19.223070","exception":false,"start_time":"2024-04-23T03:51:18.894321","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"we can see the score of each number k from log from this following box","metadata":{"papermill":{"duration":0.32468,"end_time":"2024-04-23T03:51:19.874187","exception":false,"start_time":"2024-04-23T03:51:19.549507","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# import math\n\n# k_values = [2,3,6] \n\n# for k in k_values:\n#     kmedoids = KMedoids(n_clusters=k, random_state=42)\n#     train_cluster_labels = kmedoids.fit_predict(train_features_flat)\n#     valid_cluster_labels = kmedoids.predict(valid_features_flat)\n\n#     # Assessing the validity of clustered outcome (Silhouette Score)\n#     train_silhouette_score = silhouette_score(train_features_flat, train_cluster_labels)\n#     valid_silhouette_score = silhouette_score(valid_features_flat, valid_cluster_labels)\n#     print(f\"KMedoids with k={k}: Train Silhouette Score: {train_silhouette_score}, Valid Silhouette Score: {valid_silhouette_score}\")\n\n#     # Model Fine Tuning\n#     base_model.trainable = True\n#     fine_tune_at = 100\n\n#     for layer in base_model.layers[:fine_tune_at]:\n#         layer.trainable = False\n\n#     model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001 / 10), metrics=[\"accuracy\"])\n\n#     # Calculate steps_per_epoch and validation_steps as integers\n#     steps_per_epoch = math.ceil(train_it.samples / train_it.batch_size)\n#     validation_steps = math.ceil(valid_it.samples / valid_it.batch_size)\n\n#     # Model fitting\n#     history = model.fit(train_it,\n#                         validation_data=valid_it,\n#                         steps_per_epoch=steps_per_epoch,\n#                         validation_steps=validation_steps,\n#                         epochs=5)\n","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:51:20.533459Z","iopub.status.busy":"2024-04-23T03:51:20.533064Z","iopub.status.idle":"2024-04-23T03:51:20.539011Z","shell.execute_reply":"2024-04-23T03:51:20.537981Z"},"papermill":{"duration":0.335741,"end_time":"2024-04-23T03:51:20.541146","exception":false,"start_time":"2024-04-23T03:51:20.205405","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make a submission files with k = 2,3,6\n","metadata":{"papermill":{"duration":0.388509,"end_time":"2024-04-23T03:51:21.263458","exception":false,"start_time":"2024-04-23T03:51:20.874949","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install scikit-learn-extra\n","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:51:21.937705Z","iopub.status.busy":"2024-04-23T03:51:21.937296Z","iopub.status.idle":"2024-04-23T03:51:35.472959Z","shell.execute_reply":"2024-04-23T03:51:35.471542Z"},"papermill":{"duration":13.88715,"end_time":"2024-04-23T03:51:35.475614","exception":false,"start_time":"2024-04-23T03:51:21.588464","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom os.path import isfile, join\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\n# Directory containing validation images\nvalidation_dir = '../input/csc532-fruits-categorical-classification-2024/csc532_2024_validation/validation'\n\nresult_dict = {\n    'ID': [],\n    'Code': [],\n}\n\n# Iterate through validation images\nfor file in os.listdir(validation_dir):\n    if isfile(join(validation_dir, file)):\n        # Load and preprocess image\n        image = load_img(join(validation_dir, file), target_size=(224, 224))\n        image = img_to_array(image)\n        image = np.expand_dims(image, axis=0)  # Add batch dimension\n        image = image / 255.0  # Normalize pixel values\n        \n        # Make prediction\n        prediction = np.argmax(model.predict(image)) + 1  # Assuming class indices start from 1\n\n        # Append to result_dict\n        result_dict['ID'].append(file)\n        result_dict['Code'].append(prediction)\n\n# Create DataFrame from result_dict and save to CSV\ndf = pd.DataFrame.from_dict(result_dict)\ndf.to_csv('result_kmedoids.csv', index=False)","metadata":{"execution":{"iopub.execute_input":"2024-04-23T03:51:36.152493Z","iopub.status.busy":"2024-04-23T03:51:36.152067Z","iopub.status.idle":"2024-04-23T03:52:02.370405Z","shell.execute_reply":"2024-04-23T03:52:02.369581Z"},"papermill":{"duration":26.554537,"end_time":"2024-04-23T03:52:02.372493","exception":false,"start_time":"2024-04-23T03:51:35.817956","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}