{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":87323,"databundleVersionId":9934427,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CIDAUT AI Fake Scene Classification 2024","metadata":{}},{"cell_type":"markdown","source":"---\nauthor: Brice Piaple Dada\ndate: December 21, 2024\n---","metadata":{}},{"cell_type":"markdown","source":"## Import our libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport os\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.245361Z","iopub.execute_input":"2024-12-30T21:54:32.245687Z","iopub.status.idle":"2024-12-30T21:54:32.636189Z","shell.execute_reply.started":"2024-12-30T21:54:32.245661Z","shell.execute_reply":"2024-12-30T21:54:32.635292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load our data","metadata":{}},{"cell_type":"markdown","source":"###  Load our csv and images files","metadata":{}},{"cell_type":"code","source":"csv_path = '/kaggle/input/cidaut-ai-fake-scene-classification-2024/train.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.637308Z","iopub.execute_input":"2024-12-30T21:54:32.637821Z","iopub.status.idle":"2024-12-30T21:54:32.641070Z","shell.execute_reply.started":"2024-12-30T21:54:32.637797Z","shell.execute_reply":"2024-12-30T21:54:32.640339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Charger le fichier CSV\ndf = pd.read_csv(csv_path)  # Remplacez \"train.csv\" par le nom de votre fichier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.642597Z","iopub.execute_input":"2024-12-30T21:54:32.642804Z","iopub.status.idle":"2024-12-30T21:54:32.668311Z","shell.execute_reply.started":"2024-12-30T21:54:32.642787Z","shell.execute_reply":"2024-12-30T21:54:32.667476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Afficher les premières lignes\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.669646Z","iopub.execute_input":"2024-12-30T21:54:32.669945Z","iopub.status.idle":"2024-12-30T21:54:32.683427Z","shell.execute_reply.started":"2024-12-30T21:54:32.669914Z","shell.execute_reply":"2024-12-30T21:54:32.682527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vérifier les dimensions du DataFrame\nprint(f\"Nombre de lignes: {df.shape[0]}, Nombre de colonnes: {df.shape[1]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.684320Z","iopub.execute_input":"2024-12-30T21:54:32.684584Z","iopub.status.idle":"2024-12-30T21:54:32.689810Z","shell.execute_reply.started":"2024-12-30T21:54:32.684549Z","shell.execute_reply":"2024-12-30T21:54:32.688949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vérification les types de données\ndf.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.690559Z","iopub.execute_input":"2024-12-30T21:54:32.690748Z","iopub.status.idle":"2024-12-30T21:54:32.706404Z","shell.execute_reply.started":"2024-12-30T21:54:32.690731Z","shell.execute_reply":"2024-12-30T21:54:32.705544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df[\"label\"].value_counts())\nprint(df[\"label\"].value_counts(normalize=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.707322Z","iopub.execute_input":"2024-12-30T21:54:32.707565Z","iopub.status.idle":"2024-12-30T21:54:32.732091Z","shell.execute_reply.started":"2024-12-30T21:54:32.707536Z","shell.execute_reply":"2024-12-30T21:54:32.731330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndf[\"label\"].value_counts().plot(kind='bar')\nplt.title(\"Distribution des labels\")\nplt.xlabel(\"Label\")\nplt.ylabel(\"Nombre d'images\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.732893Z","iopub.execute_input":"2024-12-30T21:54:32.733112Z","iopub.status.idle":"2024-12-30T21:54:32.989995Z","shell.execute_reply.started":"2024-12-30T21:54:32.733093Z","shell.execute_reply":"2024-12-30T21:54:32.989084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\n# Fonction pour afficher une image aléatoire d'une classe donnée\ndef visualize_images(df, label, num_images=5):\n    filtered_df = df[df[\"label\"] == label]\n    if filtered_df.empty:\n        print(f\"No images found for label {label}\")\n        return\n    image_files = filtered_df[\"image\"].sample(min(num_images, len(filtered_df))).tolist()\n    image_dir = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Train\" #Chemin vers le dossier\n\n    plt.figure(figsize=(15, 5))\n    for i, image_file in enumerate(image_files):\n        image_path = os.path.join(image_dir, image_file)\n        img = mpimg.imread(image_path)\n        plt.subplot(1, num_images, i + 1)\n        plt.imshow(img)\n        plt.title(f\"Label: {label}\")\n        plt.axis(\"off\")\n    plt.show()\n\n# Visualiser 5 images réelles (label 1)\nvisualize_images(df, 'real')\n\n# Visualiser 5 images fausses (label 0)\nvisualize_images(df, 'editada')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:32.992858Z","iopub.execute_input":"2024-12-30T21:54:32.993079Z","iopub.status.idle":"2024-12-30T21:54:35.046533Z","shell.execute_reply.started":"2024-12-30T21:54:32.993061Z","shell.execute_reply":"2024-12-30T21:54:35.045632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\n\nimage_dir = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Train\"\n\ndef get_image_sizes(df: pd.Series, image_dir: str):\n    widths = []\n    heights = []\n    for image_file in df[\"image\"]:\n        image_path = os.path.join(image_dir, image_file)\n        try:\n            with Image.open(image_path) as img:\n                widths.append(img.width)\n                heights.append(img.height)\n        except FileNotFoundError:\n            print(f\"Image not found: {image_path}\")\n    return widths, heights\n\nwidths, heights = get_image_sizes(df, image_dir)\n\nprint(f\"Largeur moyenne: {sum(widths) / len(widths):.2f}\")\nprint(f\"Hauteur moyenne: {sum(heights) / len(heights):.2f}\")\nprint(f\"Largeur min: {min(widths)}\")\nprint(f\"Largeur max: {max(widths)}\")\nprint(f\"Hauteur min: {min(heights)}\")\nprint(f\"Hauteur max: {max(heights)}\")\n\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.hist(widths, bins=50)\nplt.title(\"Distribution des largeurs\")\nplt.subplot(1, 2, 2)\nplt.hist(heights, bins=50)\nplt.title(\"Distribution des hauteurs\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:35.048059Z","iopub.execute_input":"2024-12-30T21:54:35.048383Z","iopub.status.idle":"2024-12-30T21:54:40.821184Z","shell.execute_reply.started":"2024-12-30T21:54:35.048341Z","shell.execute_reply":"2024-12-30T21:54:40.820198Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Préparation des données","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlabel_encoder = LabelEncoder()\n\n# Convertir les labels en entiers\ndf[\"label_int\"] = label_encoder.fit_transform(df[\"label\"])\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:40.821973Z","iopub.execute_input":"2024-12-30T21:54:40.822252Z","iopub.status.idle":"2024-12-30T21:54:40.831297Z","shell.execute_reply.started":"2024-12-30T21:54:40.822222Z","shell.execute_reply":"2024-12-30T21:54:40.830574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"image_path\"] = df[\"image\"].apply(lambda x: os.path.join(image_dir, x))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:40.831985Z","iopub.execute_input":"2024-12-30T21:54:40.832274Z","iopub.status.idle":"2024-12-30T21:54:40.844264Z","shell.execute_reply.started":"2024-12-30T21:54:40.832252Z","shell.execute_reply":"2024-12-30T21:54:40.843584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.2, stratify=df[\"label\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:40.844983Z","iopub.execute_input":"2024-12-30T21:54:40.845228Z","iopub.status.idle":"2024-12-30T21:54:40.860185Z","shell.execute_reply.started":"2024-12-30T21:54:40.845198Z","shell.execute_reply":"2024-12-30T21:54:40.859368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Train size: {len(train_df)}\")\nprint(f\"Validation size: {len(val_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:40.861033Z","iopub.execute_input":"2024-12-30T21:54:40.861334Z","iopub.status.idle":"2024-12-30T21:54:40.873001Z","shell.execute_reply.started":"2024-12-30T21:54:40.861307Z","shell.execute_reply":"2024-12-30T21:54:40.872160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nbatch_size = 32\nimage_size = (224, 224)\n\ntrain_datagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    shear_range=0.1,\n    zoom_range=0.1,\n    horizontal_flip=True,\n    fill_mode=\"nearest\",\n    preprocessing_function=tf.keras.applications.resnet50.preprocess_input,\n)\n\nval_datagen = ImageDataGenerator(\n    preprocessing_function=tf.keras.applications.resnet50.preprocess_input\n)\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe=train_df,\n    x_col=\"image_path\",\n    y_col=\"label_int\",\n    target_size=image_size,\n    batch_size=batch_size,\n    class_mode=\"raw\",\n    shuffle=True,\n)\n\nvalidation_generator = val_datagen.flow_from_dataframe(\n    dataframe=val_df,\n    x_col=\"image_path\",\n    y_col=\"label_int\",\n    target_size=image_size,\n    batch_size=batch_size,\n    class_mode=\"raw\",\n    shuffle=False,\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:41.083451Z","iopub.execute_input":"2024-12-30T21:54:41.083844Z","iopub.status.idle":"2024-12-30T21:54:41.315014Z","shell.execute_reply.started":"2024-12-30T21:54:41.083809Z","shell.execute_reply":"2024-12-30T21:54:41.314278Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define the model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet101\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\n\nbase_model = ResNet101(weights=\"imagenet\", include_top=False, input_shape=(224, 224, 3))\n# base_model = EfficientNetB0(weights=\"imagenet\", include_top=False, input_shape=(224, 224, 3))\n\n\nbase_model.trainable = False\nfor layer in base_model.layers[:-20]:\n    layer.trainable = False\n\n\nmodel = Sequential(\n    [\n        base_model,\n        GlobalAveragePooling2D(),\n        Dense(1, activation=\"sigmoid\"),\n    ]\n)\n\n# Compiler le modèle\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n    loss=\"binary_crossentropy\",\n    metrics=[\"accuracy\"],\n)\n\n# model.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T22:17:34.311980Z","iopub.execute_input":"2024-12-30T22:17:34.312304Z","iopub.status.idle":"2024-12-30T22:17:36.246316Z","shell.execute_reply.started":"2024-12-30T22:17:34.312277Z","shell.execute_reply":"2024-12-30T22:17:36.245612Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Fit the model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\n\n# Définir un callback EarlyStopping\nearly_stopping = EarlyStopping(\n     monitor=\"val_accuracy\", patience=5, restore_best_weights=True\n )\n\n# Entraîner le modèle\nhistory = model.fit(\n    train_generator,\n    epochs=50, # Nombre d'époques à adapter\n    validation_data=validation_generator,\n    callbacks=[early_stopping],\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T22:17:36.694008Z","iopub.execute_input":"2024-12-30T22:17:36.694333Z","iopub.status.idle":"2024-12-30T22:19:20.560003Z","shell.execute_reply.started":"2024-12-30T22:17:36.694307Z","shell.execute_reply":"2024-12-30T22:19:20.559264Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## Phase de Finalisation et Soumission\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntest_dir = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Test\" \n\ndef preprocess_image_test(image_path, image_size=(224, 224)):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, image_size)\n    img = tf.keras.applications.resnet50.preprocess_input(img)\n    return img\n\nimage_size = (224, 224)\n\n# Liste des chemins des images de test\nimage_paths = [os.path.join(test_dir, f) for f in os.listdir(test_dir) if f.endswith(('.jpg', '.jpeg'))]\nnum_samples = len(image_paths)\nprint(\"Found {} images.\".format(num_samples))\n\n# Créer un dataset à partir des chemins des images\ndataset = tf.data.Dataset.from_tensor_slices(image_paths)\n\n# Appliquer la fonction de prétraitement à chaque image\ndataset = dataset.map(lambda x: preprocess_image_test(x, image_size))\n\n# Créer des batchs de taille 1\ndataset = dataset.batch(1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T22:26:17.765714Z","iopub.execute_input":"2024-12-30T22:26:17.766010Z","iopub.status.idle":"2024-12-30T22:26:17.840693Z","shell.execute_reply.started":"2024-12-30T22:26:17.765989Z","shell.execute_reply":"2024-12-30T22:26:17.839958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Effectuer les prédictions\npredictions = model.predict(dataset, steps=num_samples)\n\n# Binariser les prédictions (si nécessaire)\nbinary_predictions = (predictions > 0.5).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T22:26:20.170983Z","iopub.execute_input":"2024-12-30T22:26:20.171293Z","iopub.status.idle":"2024-12-30T22:26:29.172862Z","shell.execute_reply.started":"2024-12-30T22:26:20.171271Z","shell.execute_reply":"2024-12-30T22:26:29.172116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extraire les noms de fichiers avec les extensions\nimage_names = [os.path.basename(f) for f in image_paths]\n# Créer un DataFrame avec les prédictions\nsubmission_df = pd.DataFrame(\n    {\"image\": image_names, \"label\": binary_predictions.flatten()}\n)\n\n# Enregistrer le DataFrame dans un fichier CSV\nsubmission_df.to_csv(\"submission.csv\", index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T22:26:29.173904Z","iopub.execute_input":"2024-12-30T22:26:29.174231Z","iopub.status.idle":"2024-12-30T22:26:29.180948Z","shell.execute_reply.started":"2024-12-30T22:26:29.174194Z","shell.execute_reply":"2024-12-30T22:26:29.180002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the model\n\nmodel.save(\"best_model.keras\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:57:26.497625Z","iopub.execute_input":"2024-12-30T21:57:26.497930Z","iopub.status.idle":"2024-12-30T21:57:27.040928Z","shell.execute_reply.started":"2024-12-30T21:57:26.497897Z","shell.execute_reply":"2024-12-30T21:57:27.039970Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Import the saved model","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:41.372880Z","iopub.status.idle":"2024-12-30T21:54:41.373271Z","shell.execute_reply":"2024-12-30T21:54:41.373083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_saved = tf.keras.models.load_model(\"/kaggle/working/best_model.h5\", compile=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T21:54:41.374111Z","iopub.status.idle":"2024-12-30T21:54:41.374506Z","shell.execute_reply":"2024-12-30T21:54:41.374341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}