{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Required Imports and Libraries**","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import ResNet50, VGG16, EfficientNetB0, EfficientNetV2B0, VGG19 # You can choose other pre-trained models as well\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Flatten\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import models, layers\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.metrics import confusion_matrix, f1_score\n\n\nimport PIL\nimport PIL.Image\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n\nimport os\nimport shutil\nimport time\nimport random\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-31T17:50:08.045111Z","iopub.execute_input":"2023-08-31T17:50:08.045717Z","iopub.status.idle":"2023-08-31T17:50:08.058734Z","shell.execute_reply.started":"2023-08-31T17:50:08.045662Z","shell.execute_reply":"2023-08-31T17:50:08.057532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def plot_loss_curves(history):\n  loss = history.history['loss']\n  val_loss = history.history['val_loss']\n\n  accuracy = history.history['accuracy']\n  val_accuracy = history.history['val_accuracy']\n\n  epochs = range(len(history.history['loss']))\n\n  # Plot loss\n  plt.plot(epochs, loss, label='training_loss')\n  plt.plot(epochs, val_loss, label='val_loss')\n  plt.title('Loss')\n  plt.xlabel('Epochs')\n  plt.legend()\n\n  # Plot accuracy\n  plt.figure()\n  plt.plot(epochs, accuracy, label='training_accuracy')\n  plt.plot(epochs, val_accuracy, label='val_accuracy')\n  plt.title('Accuracy')\n  plt.xlabel('Epochs')\n  plt.legend();\n    \n    \ndef view_random_image(target_dir, target_class):\n  # Setup target directory (we'll view images from here)\n  target_folder = target_dir+target_class\n\n  # Get a random image path\n  random_image = random.sample(os.listdir(target_folder), 1)\n\n  # Read in the image and plot it using matplotlib\n  img = mpimg.imread(target_folder + \"/\" + random_image[0])\n  plt.imshow(img)\n  plt.title(target_class)\n  plt.axis(\"off\");\n\n  print(f\"Image shape: {img.shape}\") # show the shape of the image\n\n  return img","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:46:07.603434Z","iopub.execute_input":"2023-08-31T17:46:07.603833Z","iopub.status.idle":"2023-08-31T17:46:07.614841Z","shell.execute_reply.started":"2023-08-31T17:46:07.603784Z","shell.execute_reply":"2023-08-31T17:46:07.613575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copying Images from patient folders to classes folders","metadata":{}},{"cell_type":"code","source":"source_dir = '/kaggle/input/bioverse-hackathon-a/train'\ndest_dir = '/kaggle/working/data/train'\n\nclass_folders = ['beneign', 'malignant', 'normal']\n\n# Create destination directories if they don't exist\nfor class_folder in class_folders:\n    os.makedirs(os.path.join(dest_dir, class_folder), exist_ok=True)\n\nfor class_folder in class_folders:\n    class_path = os.path.join(source_dir, class_folder)\n\n    patient_folders = os.listdir(class_path)\n\n    for patient_folder in patient_folders:\n        patient_path = os.path.join(class_path, patient_folder)\n        # image_files = [f for f in os.listdir(patient_path) if f.lower().endswith('.jpg') or f.lower().endswith('.png')]\n        image_files = os.listdir(patient_path)\n        for image_file in image_files:\n            source_image_path = os.path.join(patient_path, image_file)\n            dest_image_path = os.path.join(dest_dir, class_folder, image_file)\n            shutil.copy(source_image_path, dest_image_path)\nprint(\"Image movement complete.\")\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:41:37.992902Z","iopub.execute_input":"2023-08-31T17:41:37.993332Z","iopub.status.idle":"2023-08-31T17:41:49.372323Z","shell.execute_reply.started":"2023-08-31T17:41:37.993297Z","shell.execute_reply":"2023-08-31T17:41:49.371199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring data","metadata":{}},{"cell_type":"code","source":"source_dir = '/kaggle/input/bioverse-hackathon-a/train'\ndest_dir = '/kaggle/working/data/train'\n\nclass_folders = ['beneign', 'malignant', 'normal']\n\nimage_count = 0\nfor class_folder in class_folders:\n    imgCount =0\n    class_path = os.path.join(source_dir, class_folder)\n\n    patient_folders = os.listdir(class_path)\n\n    for patient_folder in patient_folders:\n        patient_path = os.path.join(class_path, patient_folder)\n        # image_files = [f for f in os.listdir(patient_path) if f.lower().endswith('.jpg') or f.lower().endswith('.png')]\n        image_files = os.listdir(patient_path)\n        for image_file in image_files:\n            image_count +=1\n            imgCount +=1\n    print(class_folder+' has : ' + str(imgCount) + ' images')\nprint('Total Images are : ' + str(image_count))","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:43:24.244705Z","iopub.execute_input":"2023-08-31T17:43:24.245085Z","iopub.status.idle":"2023-08-31T17:43:24.313703Z","shell.execute_reply.started":"2023-08-31T17:43:24.245054Z","shell.execute_reply":"2023-08-31T17:43:24.312657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_dir = '/kaggle/working/data/train/'  # Path to the test directory\ntarget_class1 = 'beneign'  # Change this to the desired class\ntarget_class2 = 'normal'  # Change this to the desired class\ntarget_class3 = 'malignant'  # Change this to the desired class\nplt.figure()\nplt.subplot(1,3,1)\nrandom_image1 = view_random_image(target_dir, target_class1)\nplt.subplot(1,3,2)\nrandom_image2 = view_random_image(target_dir, target_class2)\nplt.subplot(1,3,3)\nrandom_image3 = view_random_image(target_dir, target_class3)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:50:33.114738Z","iopub.execute_input":"2023-08-31T17:50:33.115114Z","iopub.status.idle":"2023-08-31T17:50:35.054377Z","shell.execute_reply.started":"2023-08-31T17:50:33.115085Z","shell.execute_reply":"2023-08-31T17:50:35.053194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting Data","metadata":{}},{"cell_type":"code","source":"train_data_dir = '/kaggle/working/data/train'\n\ntrain_datagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    validation_split=0.2)  # Split 20% of data for validation\n\ntrain_generator = train_datagen.flow_from_directory(\n    train_data_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical',\n    subset='training')  # Use the training subset\n\nvalidation_generator = train_datagen.flow_from_directory(\n    train_data_dir,\n    target_size=(224, 224),\n    batch_size=32,\n    class_mode='categorical',\n    subset='validation')  # Use the validation subset\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T15:58:32.330927Z","iopub.execute_input":"2023-08-31T15:58:32.331319Z","iopub.status.idle":"2023-08-31T15:58:32.365032Z","shell.execute_reply.started":"2023-08-31T15:58:32.331281Z","shell.execute_reply":"2023-08-31T15:58:32.364098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building The Model","metadata":{}},{"cell_type":"code","source":"base_model = EfficientNetB0( include_top=False)\n\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\nx = Dense(256, activation='relu')(x)\nx = Dense(32, activation='relu')(x)\npredictions = Dense(3, activation='softmax')(x)\n\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\nfor layer in base_model.layers:\n    layer.trainable = True\n\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()\nhistory = model.fit(train_generator, epochs=10, validation_data=validation_generator)  # Fine-tune for a suitable number of epochs\ntest_loss, test_acc = model.evaluate(validation_generator)\nprint(f\"Test accuracy: {test_acc}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:33:11.901398Z","iopub.execute_input":"2023-08-31T17:33:11.901817Z","iopub.status.idle":"2023-08-31T17:39:19.960795Z","shell.execute_reply.started":"2023-08-31T17:33:11.901786Z","shell.execute_reply":"2023-08-31T17:39:19.95948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting some statistics","metadata":{}},{"cell_type":"code","source":"plot_loss_curves(history)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:39:19.963031Z","iopub.execute_input":"2023-08-31T17:39:19.963794Z","iopub.status.idle":"2023-08-31T17:39:20.628436Z","shell.execute_reply.started":"2023-08-31T17:39:19.963739Z","shell.execute_reply":"2023-08-31T17:39:20.627421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and submission","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nimport numpy as np\n    \nclass_labels = ['beneign', 'malignant', 'normal']\n\n# Path to the folder containing images for prediction\nprediction_folder = '/kaggle/input/bioverse-hackathon-a/test'\n# Load the trained model\n# List of image filenames in the prediction folder\nimage_filenames = os.listdir(prediction_folder)\n# List to store predictions\npredictions = []\n# Class labels for mapping predicted class indices to class names\n# class_labels = ['normal', 'beneign', 'malignant']\n# Loop through each image and make predictions\nfor filename in image_filenames:\n    image_path = os.path.join(prediction_folder, filename)\n    img = load_img(image_path, target_size=(224, 224))\n    img_array = img_to_array(img)\n    #img_array = img_array / 255.0  # Normalize pixel values\n    img_array = np.expand_dims(img_array, axis=0)\n    # Add batch dimension\n    prediction = model.predict(img_array)\n    print(prediction)\n    predicted_class_index = np.argmax(prediction)\n    predicted_class = class_labels[predicted_class_index]\n    predictions.append((filename, predicted_class))\n# Create a DataFrame and save it as a CSV file\npredictions_df = pd.DataFrame(predictions, columns=['filename', 'class'])\npredictions_csv_path = 'predictions.csv'\npredictions_df.to_csv(predictions_csv_path, index=False)\nprint(f'Predictions saved to {predictions_csv_path}')\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T16:45:52.67176Z","iopub.execute_input":"2023-08-31T16:45:52.672156Z","iopub.status.idle":"2023-08-31T16:46:17.726216Z","shell.execute_reply.started":"2023-08-31T16:45:52.672126Z","shell.execute_reply":"2023-08-31T16:46:17.725258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Sample ","metadata":{}},{"cell_type":"code","source":"selected_filenames = np.random.choice(image_filenames, size=1, replace=False)\nprint(selected_filenames)","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:18:15.192463Z","iopub.execute_input":"2023-08-31T17:18:15.193494Z","iopub.status.idle":"2023-08-31T17:18:15.199236Z","shell.execute_reply.started":"2023-08-31T17:18:15.193456Z","shell.execute_reply":"2023-08-31T17:18:15.198157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking Inference Time for a random sample","metadata":{}},{"cell_type":"code","source":"inference_times = []\nfor filename in selected_filenames:\n    image_path = os.path.join(prediction_folder, filename)\n    img = load_img(image_path, target_size=(224, 224))\n    img_array = img_to_array(img)\n    img_array = np.expand_dims(img_array, axis=0)\n    \n    start_time = time.time()\n    prediction = model.predict(img_array)\n    end_time = time.time()\n    \n    inference_time = end_time - start_time\n    inference_times.append(inference_time)\n    plt.figure(figsize=(6, 6))\n    plt.imshow(img)\n    plt.title(f\"Predicted Class: {predicted_class}\\nInference Time: {inference_time:.4f} seconds\")\n    plt.axis('off')\n    plt.show()\n    print(f\"Inference time for {filename}: {inference_time:.4f} seconds\")\n","metadata":{"execution":{"iopub.status.busy":"2023-08-31T17:21:24.867474Z","iopub.execute_input":"2023-08-31T17:21:24.868498Z","iopub.status.idle":"2023-08-31T17:21:25.27856Z","shell.execute_reply.started":"2023-08-31T17:21:24.86846Z","shell.execute_reply":"2023-08-31T17:21:25.277462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}