{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":67836,"databundleVersionId":7527339,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Necessary Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nfrom decimal import Decimal\nimport csv\nimport random\nfrom PIL import Image\nimport cv2\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-02T17:03:58.433134Z","iopub.execute_input":"2024-02-02T17:03:58.433875Z","iopub.status.idle":"2024-02-02T17:03:58.439041Z","shell.execute_reply.started":"2024-02-02T17:03:58.433842Z","shell.execute_reply":"2024-02-02T17:03:58.437917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Load Data","metadata":{}},{"cell_type":"code","source":"path_data='/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD'\ntrain_label='labels/train'\ntest_image='images/test'\ntrain_image='images/train'\nclass_path='/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/badodd.txt'","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:55:43.885620Z","iopub.execute_input":"2024-02-02T16:55:43.886106Z","iopub.status.idle":"2024-02-02T16:55:43.891361Z","shell.execute_reply.started":"2024-02-02T16:55:43.886072Z","shell.execute_reply":"2024-02-02T16:55:43.890250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img_list=os.listdir(os.path.join(path_data,train_image))\ntrain_label_list=os.listdir(os.path.join(path_data,train_label))\ntest_img_list=os.listdir(os.path.join(path_data,test_image))\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:55:43.897567Z","iopub.execute_input":"2024-02-02T16:55:43.897976Z","iopub.status.idle":"2024-02-02T16:55:44.424427Z","shell.execute_reply.started":"2024-02-02T16:55:43.897943Z","shell.execute_reply":"2024-02-02T16:55:44.423440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Overview of Dataset\nThe dataset covers the following 9 districts in Bangladesh: Sylhet, Dhaka, Rajshahi, Mymensingh, Maowa, Chittagong, Sirajganj, Sherpur, and Khulna. This dataset contains 9825 images with 78,943 Objects with 13 different classes. It is annotated with rectangular bounding box.<br>\nClasses : 13<br>\nGoal: **Developing solutions to detect objects in various Road Conditions of Bangladesh.**\n\n","metadata":{}},{"cell_type":"code","source":"print(\"Training samples:\",len(train_img_list))\nprint(\"Testing samples:\",len(test_img_list))\nprint(\"Training labels:\",len(train_label_list))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:55:44.426044Z","iopub.execute_input":"2024-02-02T16:55:44.426431Z","iopub.status.idle":"2024-02-02T16:55:44.432448Z","shell.execute_reply.started":"2024-02-02T16:55:44.426393Z","shell.execute_reply":"2024-02-02T16:55:44.431568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classNames={};\nwith open(class_path) as classes:\n    lines=classes.read();\n    lines=lines.split('\\n');\n    for i in range(len(lines)):\n        line=lines[i];\n        if len(line)==0:\n            continue;\n        classNames[i]=line;\nprint(f\"There are {len(classNames)} classes!\")\nclassNames","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:55:44.433566Z","iopub.execute_input":"2024-02-02T16:55:44.434330Z","iopub.status.idle":"2024-02-02T16:55:44.453132Z","shell.execute_reply.started":"2024-02-02T16:55:44.434299Z","shell.execute_reply":"2024-02-02T16:55:44.452109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classCounts={};\nfor i in range(len(train_label_list)):\n    with open(os.path.join(path_data,train_label,train_label_list[i])) as file:\n        lines=file.read().split('\\n');\n        for line in lines:\n            if len(line)==0:\n                continue;\n            key=line.split()[0];\n            \n            if key in classCounts.keys():\n                classCounts[key]+=1;\n            else:\n                classCounts[key]=1;\nlabels=[];\ncounts=[];\nfor i in classCounts.keys():\n    print(classNames[int(i)], classCounts[i])\n    labels.append(classNames[int(i)])\n    counts.append(classCounts[i])","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:55:44.454611Z","iopub.execute_input":"2024-02-02T16:55:44.455238Z","iopub.status.idle":"2024-02-02T16:56:09.577693Z","shell.execute_reply.started":"2024-02-02T16:55:44.455204Z","shell.execute_reply":"2024-02-02T16:56:09.576798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.xticks(rotation=45, ha='right')\nbars = plt.bar(labels,counts,width = 0.5)\nfor bar in bars:\n    plt.text(bar.get_x() + bar.get_width() / 2, bar.get_height(), int(bar.get_height()), va='bottom', ha='center')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:09.578774Z","iopub.execute_input":"2024-02-02T16:56:09.579021Z","iopub.status.idle":"2024-02-02T16:56:09.902999Z","shell.execute_reply.started":"2024-02-02T16:56:09.578999Z","shell.execute_reply":"2024-02-02T16:56:09.902175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Exploring Images and Labels","metadata":{}},{"cell_type":"code","source":"def create_csv_from_directory(directory, output_filename):\n    with open(output_filename, 'w', newline='') as f:\n        writer = csv.writer(f)\n        writer.writerow(['filename', 'id', 'x_center', 'y_center', 'width', 'height'])\n\n        # Get list of all text files in the directory\n        files = [f for f in os.listdir(directory) if f.endswith('.txt')]\n\n        for i, filename in enumerate(files):\n            with open(os.path.join(directory, filename), 'r') as f:\n                lines = f.readlines()\n\n            for j, line in enumerate(lines):\n                parts = line.strip().split() # Remove trailing newline and split by spaces\n                key = int(parts[0]) # Convert the first part to an integer\n                x_center, y_center, width, height = map(float, parts[1:]) # Convert the rest to floats\n                writer.writerow([filename.replace('.txt',''), key, x_center, y_center, width, height])","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:09.904191Z","iopub.execute_input":"2024-02-02T16:56:09.904570Z","iopub.status.idle":"2024-02-02T16:56:09.912697Z","shell.execute_reply.started":"2024-02-02T16:56:09.904537Z","shell.execute_reply":"2024-02-02T16:56:09.911831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(_image_id):\n    image_path = '/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train/'+ _image_id +'.jpg'\n    image = Image.open(image_path)\n    plt.figure(figsize=(15, 15))\n    plt.axis('off')\n    plt.imshow(np.asarray(image))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:09.916077Z","iopub.execute_input":"2024-02-02T16:56:09.916396Z","iopub.status.idle":"2024-02-02T16:56:09.929191Z","shell.execute_reply.started":"2024-02-02T16:56:09.916360Z","shell.execute_reply":"2024-02-02T16:56:09.928334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_segmentations(_image_id):\n    filtered_df = train_df[train_df['filename'] == _image_id]\n#     print(filtered_df)\n    image = np.array(Image.open('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train/' + _image_id + '.jpg'))\n    class_colors = {\n        0: (255, 0, 0),      # Red\n        1: (0, 255, 0),      # Green\n        2: (0, 0, 255),      # Blue\n        3: (255, 255, 0),    # Yellow\n        4: (0, 255, 255),    # Cyan\n        5: (255, 0, 255),    # Magenta\n        6: (128, 0, 0),      # Dark Red\n        7: (0, 128, 0),      # Dark Green\n        8: (0, 0, 128),      # Dark Blue\n        9: (128, 128, 0),    # Olive\n        10: (0, 128, 128),   # Teal\n        11: (128, 0, 128),   # Purple\n        12: (128, 128, 128)  # Grey\n    }\n    \n    for _, row in filtered_df.iterrows():\n        x_center, y_center, width, height = row['x_center'], row['y_center'], row['width'], row['height']\n        class_num = row['id']\n        x_center, y_center, width, height = int(x_center * image.shape[1]), int(y_center * image.shape[0]), int(width * image.shape[1]), int(height * image.shape[0])\n        xmin, ymin = x_center - width // 2, y_center - height // 2\n        xmax, ymax = x_center + width // 2, y_center + height // 2\n        color = class_colors[class_num]\n        cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color, 2)\n        title = f\"{str(row['id'])} ({classNames[row['id']]})\"\n        cv2.putText(image, title, (xmin, ymin - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (36,255,12), 2)\n    \n    plt.figure(figsize=(15, 15))\n    plt.axis('off')\n    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:09.930268Z","iopub.execute_input":"2024-02-02T16:56:09.931079Z","iopub.status.idle":"2024-02-02T16:56:09.943061Z","shell.execute_reply.started":"2024-02-02T16:56:09.931053Z","shell.execute_reply":"2024-02-02T16:56:09.942288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.1 Loding in DataFrame","metadata":{}},{"cell_type":"code","source":"create_csv_from_directory('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/labels/train', 'train.csv')\ntrain_df = pd.read_csv('/kaggle/working/train.csv')\nIMG_PATH= '/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/'\ntrain_df['image_path'] = f'{IMG_PATH}/train/' + train_df['filename'] + '.jpg'\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:09.944123Z","iopub.execute_input":"2024-02-02T16:56:09.944564Z","iopub.status.idle":"2024-02-02T16:56:13.632098Z","shell.execute_reply.started":"2024-02-02T16:56:09.944533Z","shell.execute_reply":"2024-02-02T16:56:13.631202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.2 Showing Images","metadata":{}},{"cell_type":"code","source":"image_paths = list(train_df['image_path'])\nplt.figure(figsize=(15, 15))\nfor i in range(6):\n    image_id = random.randint(2, 20000)\n    show_segmentations((train_df['filename'][image_id]))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:13.633141Z","iopub.execute_input":"2024-02-02T16:56:13.633435Z","iopub.status.idle":"2024-02-02T16:56:18.608857Z","shell.execute_reply.started":"2024-02-02T16:56:13.633406Z","shell.execute_reply":"2024-02-02T16:56:18.607946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. About Images","metadata":{}},{"cell_type":"markdown","source":"## 5.1 Image Dimension","metadata":{}},{"cell_type":"code","source":"!pip install hvplot\nfrom dask import bag, diagnostics \nimport hvplot.pandas \n\n# get image dimensions\ndef get_dims(file):\n    img = cv2.imread(file)\n    h,w = img.shape[:2]\n    return h,w\n\n# parallelize\nfilelist = set(list(train_df['image_path']))\ndimsbag = bag.from_sequence(filelist).map(get_dims)\nwith diagnostics.ProgressBar():\n    dims = dimsbag.compute()\n    \ndim_df = pd.DataFrame(dims, columns=['height', 'width'])\nsizes = dim_df.groupby(['height', 'width']).size().reset_index().rename(columns={0:'count'})\nsizes.hvplot.scatter(x='height', y='width', size='count', xlim=(0,2000), ylim=(0,3000), grid=True, xticks=2, \n        yticks=2, height=500, width=600).options(scaling_factor=0.1, line_alpha=1, fill_alpha=0)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:56:18.610189Z","iopub.execute_input":"2024-02-02T16:56:18.610537Z","iopub.status.idle":"2024-02-02T16:57:49.331066Z","shell.execute_reply.started":"2024-02-02T16:56:18.610507Z","shell.execute_reply":"2024-02-02T16:57:49.330131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. About Classes","metadata":{}},{"cell_type":"markdown","source":"## 6.1 Denisty of Class Per Image","metadata":{}},{"cell_type":"code","source":"def create_bar_chart_class_wise_density(class_no):\n    #-1 gives graph for all classed combined, 0-12 gives graph for individual class\n    classnum = []\n    for i in range(len(train_label_list)):\n        with open(os.path.join(path_data,train_label,train_label_list[i])) as file:\n            lines = file.read().split('\\n')\n            count = 0\n            for line in lines:\n                if len(line)==0:\n                    continue;\n                key=line.split()[0]\n                if key == str(class_no):\n                    count = count +1\n                if class_no == -1:\n                    count = count +1\n            classnum.append(count)\n    counts_classno = Counter(classnum)\n    bars = plt.bar(counts_classno.keys(), counts_classno.values())\n    for bar in bars:\n        plt.text(bar.get_x() + bar.get_width() / 2, bar.get_height(), int(bar.get_height()), va='bottom', ha='center')\n\n    if class_no == -1:\n        plt.title(\"Barchart for All Class Per Image\")\n    else:\n        plt.title(\"Barchart for Class: {}\".format(classNames[class_no]))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:57:49.332498Z","iopub.execute_input":"2024-02-02T16:57:49.333168Z","iopub.status.idle":"2024-02-02T16:57:49.342441Z","shell.execute_reply.started":"2024-02-02T16:57:49.333133Z","shell.execute_reply":"2024-02-02T16:57:49.341209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_bar_chart_class_wise_density(-1)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:57:49.343709Z","iopub.execute_input":"2024-02-02T16:57:49.344296Z","iopub.status.idle":"2024-02-02T16:57:52.636425Z","shell.execute_reply.started":"2024-02-02T16:57:49.344261Z","shell.execute_reply":"2024-02-02T16:57:52.635504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.2 Density of Specific Class Per Image","metadata":{}},{"cell_type":"code","source":"for i in range(13):\n    create_bar_chart_class_wise_density(i)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:57:52.637832Z","iopub.execute_input":"2024-02-02T16:57:52.638114Z","iopub.status.idle":"2024-02-02T16:58:37.008885Z","shell.execute_reply.started":"2024-02-02T16:57:52.638090Z","shell.execute_reply":"2024-02-02T16:58:37.007851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.3 Average Area per class","metadata":{}},{"cell_type":"code","source":"train_df['area'] = train_df['width']* train_df['height']\naverage_area = train_df.groupby('id')['area'].mean()\naverage_area.index = average_area.index.map(classNames)\naverage_area.plot(kind='bar', title='Average Area per Class', xlabel='Class Name', ylabel='Average Area')\nplt.xticks(rotation=45, ha='right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:58:37.010079Z","iopub.execute_input":"2024-02-02T16:58:37.010370Z","iopub.status.idle":"2024-02-02T16:58:37.265253Z","shell.execute_reply.started":"2024-02-02T16:58:37.010345Z","shell.execute_reply":"2024-02-02T16:58:37.264354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.4 Create Density Map","metadata":{}},{"cell_type":"code","source":"def create_density_map(class_no=-1, dftemp=train_df, grid_size=100):\n    if class_no != -1:\n        dftemp = dftemp[dftemp['id']==class_no]\n\n    x_min, x_max = dftemp['x_center'].min(), dftemp['x_center'].max()\n    y_min, y_max = dftemp['y_center'].min(), dftemp['y_center'].max()\n\n    x_bins = np.linspace(x_min, x_max, grid_size + 1)\n    y_bins = np.linspace(y_min, y_max, grid_size + 1)\n\n    dftemp['x_bin'] = pd.cut(dftemp['x_center'], bins=x_bins, duplicates=\"drop\")\n    dftemp['y_bin'] = pd.cut(dftemp['y_center'], bins=y_bins, duplicates=\"drop\")\n\n    density = dftemp.groupby(['x_bin', 'y_bin']).size().reset_index(name='density')\n\n    density_pivot = density.pivot(index='x_bin', columns='y_bin', values='density')\n\n    plt.figure(figsize=(15, 15))\n    plt.imshow(density_pivot.fillna(0), origin='lower', cmap='hot')\n    plt.title(f\"{classNames[class_no]} Density Graph\")\n    plt.colorbar(label='Density')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:58:37.266453Z","iopub.execute_input":"2024-02-02T16:58:37.266738Z","iopub.status.idle":"2024-02-02T16:58:37.275437Z","shell.execute_reply.started":"2024-02-02T16:58:37.266714Z","shell.execute_reply":"2024-02-02T16:58:37.274577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(13):\n    create_density_map(i, train_df, 10)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:58:37.276882Z","iopub.execute_input":"2024-02-02T16:58:37.277217Z","iopub.status.idle":"2024-02-02T16:58:43.248812Z","shell.execute_reply.started":"2024-02-02T16:58:37.277186Z","shell.execute_reply":"2024-02-02T16:58:43.247877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. About Classes","metadata":{}},{"cell_type":"code","source":"def show_image_portion_class(row):\n    base_path = '/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train/'\n    image_path = base_path + row['filename'] + '.jpg'\n    image= np.array(Image.open(image_path))\n    \n    x_center, y_center, width, height = row['x_center'], row['y_center'], row['width'], row['height']\n    x_center, y_center, width, height = int(x_center * image.shape[1]), int(y_center * image.shape[0]), int(width * image.shape[1]), int(height * image.shape[0])\n    xmin, ymin = x_center - width // 2, y_center - height // 2\n    xmax, ymax = x_center + width // 2, y_center + height // 2\n    \n    if xmin >= xmax or ymin >= ymax:\n        print(f\"Invalid cropping box for image {_image_id}: ({xmin}, {ymin}, {xmax}, {ymax})\")\n        return\n    image = image[ymin:ymax, xmin:xmax]\n    return image","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:58:43.249927Z","iopub.execute_input":"2024-02-02T16:58:43.250206Z","iopub.status.idle":"2024-02-02T16:58:43.257844Z","shell.execute_reply.started":"2024-02-02T16:58:43.250182Z","shell.execute_reply":"2024-02-02T16:58:43.256835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport random\n\ndef show_images_per_class(train_df):\n    for i in range(13):\n        print(f'Showing Class: {classNames[i]}\\n')\n        filtered_df = train_df[train_df['id'] == i]\n        it = min(len(filtered_df)-1, 10)\n        fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(10, 10))\n        axes = axes.flatten()\n        for j in range(it):\n            image_id = random.randint(0, len(filtered_df)-1)\n            image = show_image_portion_class(filtered_df.iloc[image_id])\n            if image.size == 0: # Skip if the image is empty\n                continue\n            title = str(filtered_df.iloc[image_id]['filename'])\n            axes[j].imshow(image)\n            axes[j].set_title(title)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:58:43.258928Z","iopub.execute_input":"2024-02-02T16:58:43.259574Z","iopub.status.idle":"2024-02-02T16:58:43.277800Z","shell.execute_reply.started":"2024-02-02T16:58:43.259527Z","shell.execute_reply":"2024-02-02T16:58:43.277047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images_per_class(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:58:43.278716Z","iopub.execute_input":"2024-02-02T16:58:43.278958Z","iopub.status.idle":"2024-02-02T16:59:03.608653Z","shell.execute_reply.started":"2024-02-02T16:58:43.278936Z","shell.execute_reply":"2024-02-02T16:59:03.607715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. About Environment (Day vs Night)","metadata":{}},{"cell_type":"markdown","source":"## 8.1 Image Color Intensity Distribution","metadata":{}},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\ndef calculate_brightness(image_path):\n    image = cv2.imread(image_path)\n    grayscale_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    return np.mean(grayscale_image)\n\n# Assuming train_df is your DataFrame with a column 'image_path'\nimage_paths = list(set(train_df['image_path']))\n\n# Calculate brightness for each image with tqdm\nbrightness_values = []\nfor path in tqdm(image_paths, desc='Calculating Brightness', unit='image'):\n    brightness_values.append(calculate_brightness(path))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T16:59:03.609897Z","iopub.execute_input":"2024-02-02T16:59:03.610161Z","iopub.status.idle":"2024-02-02T17:01:34.388109Z","shell.execute_reply.started":"2024-02-02T16:59:03.610138Z","shell.execute_reply":"2024-02-02T17:01:34.387189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the brightness values\nplt.plot(brightness_values, marker='o', linestyle='-', color='gray')\n\nplt.title('Image Brightness')\nplt.xlabel('Image Index')\nplt.ylabel('Brightness')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:01:34.389514Z","iopub.execute_input":"2024-02-02T17:01:34.390160Z","iopub.status.idle":"2024-02-02T17:01:34.721854Z","shell.execute_reply.started":"2024-02-02T17:01:34.390124Z","shell.execute_reply":"2024-02-02T17:01:34.720824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot histogram and overlay a curve\nplt.hist(brightness_values, bins=30, color='gray', alpha=0.7, edgecolor='black')\n# plt.plot(brightness_values, marker='', linestyle='-', color='red', label='Curve')\n\nplt.title('Image Brightness Histogram with Curve')\nplt.xlabel('Brightness')\nplt.ylabel('Frequency')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:01:34.722958Z","iopub.execute_input":"2024-02-02T17:01:34.723318Z","iopub.status.idle":"2024-02-02T17:01:35.052499Z","shell.execute_reply.started":"2024-02-02T17:01:34.723284Z","shell.execute_reply":"2024-02-02T17:01:35.051631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.2 Day vs Night","metadata":{}},{"cell_type":"code","source":"filtered_df = train_df[~train_df['filename'].str.contains('night', case=False)]\nit = min(len(filtered_df)-1, 8)\nfig, axes = plt.subplots(nrows=4, ncols=2, figsize=(10, 10))\naxes = axes.flatten()\nfor j in range(it):\n    image_id = random.randint(0, len(filtered_df)-1)\n    image= np.array(Image.open(filtered_df.iloc[image_id]['image_path']))\n    if image.size == 0:\n        continue\n    title = str(filtered_df.iloc[image_id]['filename'])\n    axes[j].imshow(image)\n    axes[j].set_title(title)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:01:35.053852Z","iopub.execute_input":"2024-02-02T17:01:35.054345Z","iopub.status.idle":"2024-02-02T17:01:38.396293Z","shell.execute_reply.started":"2024-02-02T17:01:35.054309Z","shell.execute_reply":"2024-02-02T17:01:38.395406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport torchvision.transforms.functional as F\nimport random\n\ndef Adjust_contrast(image, factor):\n    return F.adjust_contrast(image, factor)\n\ndef Adjust_brightness(image, factor):\n    return F.adjust_brightness(image, factor)\n\ndef Adjust_saturation(image, factor):\n    return F.adjust_saturation(image, factor)\n\ndef lighting_noise(image, noise_factor):\n    new_image = image\n    perms = ((0, 1, 2), (0, 2, 1), (1, 0, 2), \n             (1, 2, 0), (2, 0, 1), (2, 1, 0))\n    swap = perms[random.randint(0, len(perms)- 1)]\n    new_image = F.to_tensor(new_image)\n    new_image = new_image[swap, :, :]\n    new_image = F.to_pil_image(new_image)\n    noise = Image.new('L', new_image.size)\n    noise_data = [random.randint(0, 255) for _ in range(new_image.size[0] * new_image.size[1])]\n    noise.putdata(noise_data)\n    noise = noise.point(lambda x: x * noise_factor / 255.0)\n    new_image = Image.blend(new_image, noise.convert('RGB'), noise_factor)\n\n    return new_image\n\ndef night_to_day(image, contrast_factor=2, brightness_factor=2, saturation_factor=2, noise_factor=0.1):\n    image = Adjust_contrast(image, contrast_factor)\n    image = Adjust_brightness(image, brightness_factor)\n    image = Adjust_saturation(image, saturation_factor)\n    image = lighting_noise(image, noise_factor)\n\n    return image","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:01:38.401218Z","iopub.execute_input":"2024-02-02T17:01:38.401536Z","iopub.status.idle":"2024-02-02T17:01:41.855406Z","shell.execute_reply.started":"2024-02-02T17:01:38.401509Z","shell.execute_reply":"2024-02-02T17:01:41.854426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df = train_df[train_df['filename'].str.contains('night', case=False)]\nit = min(len(filtered_df)-1, 8)\nfig, axes = plt.subplots(nrows=4, ncols=2, figsize=(10, 10))\naxes = axes.flatten()\nfor j in range(it):\n    image_id = random.randint(0, len(filtered_df)-1)\n    image= np.array(Image.open(filtered_df.iloc[image_id]['image_path']))\n    if image.size == 0:\n        continue\n    title = str(filtered_df.iloc[image_id]['filename'])\n    axes[j].imshow(image)\n    axes[j].set_title(title)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:02:58.676259Z","iopub.execute_input":"2024-02-02T17:02:58.676634Z","iopub.status.idle":"2024-02-02T17:03:02.084222Z","shell.execute_reply.started":"2024-02-02T17:02:58.676604Z","shell.execute_reply":"2024-02-02T17:03:02.083298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncount_night_filenames = sum(1 for filename in train_img_list if 'night' in filename)\nprint(f\"Number of Night Image Training: {count_night_filenames}\")\nprint(f\"Number of Day Image Training: {5896-count_night_filenames}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:03:12.731998Z","iopub.execute_input":"2024-02-02T17:03:12.732618Z","iopub.status.idle":"2024-02-02T17:03:12.738650Z","shell.execute_reply.started":"2024-02-02T17:03:12.732585Z","shell.execute_reply":"2024-02-02T17:03:12.737751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_night_elements = sum(1 for filename in test_img_list if 'night' in filename)\n\nprint(f\"Number of Night Image Testing: {count_night_elements}\")\nprint(f\"Number of Day Image Testing: {1964 - count_night_elements}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:03:15.396401Z","iopub.execute_input":"2024-02-02T17:03:15.397040Z","iopub.status.idle":"2024-02-02T17:03:15.402401Z","shell.execute_reply.started":"2024-02-02T17:03:15.397009Z","shell.execute_reply":"2024-02-02T17:03:15.401516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 9. About Place","metadata":{}},{"cell_type":"code","source":"def remove_trailing_numbers(string):\n    pattern = re.compile(r'_\\d+$')\n    match = re.search(pattern, string)\n    if match:\n        result = string[:-len(match.group())]\n        return result\n    else:\n        return string\n\ndef analyze_and_plot_prefix(train_df, prefix):\n    new_df = train_df[train_df['filename'].str.startswith(prefix)]\n    id_value_counts = new_df['id'].value_counts().sort_index()\n    all_classes = range(13)\n    id_value_counts = id_value_counts.reindex(all_classes, fill_value=0)\n    plt.figure(figsize=(10, 6))\n    bars = plt.bar(id_value_counts.index, id_value_counts, color='skyblue')\n    for bar, count in zip(bars, id_value_counts):\n        plt.text(bar.get_x() + bar.get_width() / 2, bar.get_height(), str(count), ha='center', va='bottom')\n\n    plt.title(f'Class Distribution for {prefix}')\n    plt.xlabel('Class ID')\n    plt.ylabel('Count')\n    plt.show()\n\n\ndef count_files_starting_with(directory_path):\n    filenames_without_extension = [os.path.splitext(filename)[0] for filename in os.listdir(directory_path)]\n    string_array = [remove_trailing_numbers(s) for s in filenames_without_extension]\n    filenames = os.listdir(directory_path)\n\n    counts = {search_string: sum(filename.startswith(search_string) for filename in filenames) for search_string in string_array}\n    total_files = len(filenames)\n    sorted_counts = sorted(counts.items(), key=lambda x: (x[1] / total_files), reverse=True)\n    \n\n    for search_string, count in sorted_counts:\n        percentage = (count / total_files) * 100\n        print(f\"'{search_string}': {percentage:.4f}% Count: {count}\")\n    return sorted_counts","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:07:50.103079Z","iopub.execute_input":"2024-02-02T17:07:50.103978Z","iopub.status.idle":"2024-02-02T17:07:50.115592Z","shell.execute_reply.started":"2024-02-02T17:07:50.103944Z","shell.execute_reply":"2024-02-02T17:07:50.114677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('******************Train******************')\ndirectory_path = \"/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train\"\ntraincounts = count_files_starting_with(directory_path)\nfor a, b in traincounts:\n    analyze_and_plot_prefix(train_df, a)\n\nprint('********************Test********************')\ndirectory_path = \"/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/test\"\ntestcounts =count_files_starting_with(directory_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def see_place(prefix, key=0):\n    print(prefix)\n    print(\"Train Images\")\n    if (key==1):\n        print('No Train Appearances')\n    else:\n        \n        new_df = train_df[train_df['filename'].str.startswith(prefix)]\n        plt.figure(figsize=(15, 8))\n        for i in range(6):\n            plt.subplot(2, 3, i+1)\n            image_id = random.randint(0, len(new_df) - 1)\n            image = Image.open('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train/'+new_df['filename'].iloc[image_id]+'.jpg')\n            plt.axis('off')\n            plt.imshow(np.asarray(image))\n    \n        plt.show()\n    \n    print(\"Test Images\")\n    if(key==2):\n        print('No Test Appearances')\n    else:\n        new_list = [s for s in test_img_list if s.startswith(prefix)]\n        plt.figure(figsize=(15, 15))\n        for i in range(6):\n            plt.subplot(2, 3, i+1)\n            image_id = random.randint(0, len(new_list) - 1)\n            image = Image.open('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/test/'+new_list[image_id])\n            plt.axis('off')\n            plt.imshow(np.asarray(image))\n    \n        plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traincounts = [item[0] for item in traincounts]\ntestcounts = [item[0] for item in testcounts]\n\ndone = set()\n\nfor co in traincounts:\n    key = 2 if co not in testcounts else 0\n    see_place(prefix=co, key=key)\n    done.add(co)\n\nfor co in set(testcounts) - done:\n    see_place(prefix=co, key=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T17:09:31.839657Z","iopub.execute_input":"2024-02-02T17:09:31.839986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Got Some Help From\n[1] https://www.kaggle.com/code/brainiac2677/badlad-a-complete-eda <br>\n[2] https://www.kaggle.com/code/jpmiller/basic-eda-with-images","metadata":{}}]}