{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" # Chats et chiens: classification","metadata":{}},{"cell_type":"markdown","source":"## Importer les librairies","metadata":{}},{"cell_type":"code","source":"# pour l'extraction du dataSet:\nfrom zipfile import ZipFile\n\n# pour visualiser et gérer les données\nimport os\nimport random\n\n# pour visualiser les données et faire des graphiques\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\n# pour ouvrir des images\nimport cv2\n\n# pour visualiser et manipuler le dataset et les données\nimport pandas as pd\n\n# avoir des barre de progression dans les boucles\nfrom tqdm import tqdm\n\n# gestion des vecteurs et matrices\nimport numpy as np\n\n# pour diviser le training set et ainsi ajouter un set de validation\nfrom sklearn.model_selection import train_test_split\n\n# pour la fabrication du model de ML\nfrom tensorflow.keras import Sequential\nimport tensorflow.keras.layers as layers\nfrom tensorflow.keras.losses import CategoricalCrossentropy \nimport tensorflow.keras.callbacks as callbacks\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:22:06.753678Z","iopub.execute_input":"2023-09-02T11:22:06.754063Z","iopub.status.idle":"2023-09-02T11:22:06.761496Z","shell.execute_reply.started":"2023-09-02T11:22:06.754034Z","shell.execute_reply":"2023-09-02T11:22:06.760369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Préparation et vérification du dataset","metadata":{}},{"cell_type":"markdown","source":"### extraire les données compressés","metadata":{}},{"cell_type":"code","source":"trainSetZip_path = \"/kaggle/input/dogs-vs-cats/train.zip\"\ntestSetZip_path = \"/kaggle/input/dogs-vs-cats/test1.zip\"\n\nwith ZipFile(trainSetZip_path, \"r\") as trainFile:\n    trainFile.extractall()\n    print(\"trainSetFile extracted\")\n    \nwith ZipFile(testSetZip_path, \"r\") as testFile:\n    testFile.extractall()\n    print(\"testSetFile extracted\")","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:22:26.637343Z","iopub.execute_input":"2023-09-02T11:22:26.637690Z","iopub.status.idle":"2023-09-02T11:22:43.538569Z","shell.execute_reply.started":"2023-09-02T11:22:26.637661Z","shell.execute_reply":"2023-09-02T11:22:43.537578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### compter le nombre d'images du trainset et du testset","metadata":{}},{"cell_type":"code","source":"#os.walk() : renvoie le chemin d'accès tout les sous répértoirs et le nom de chaque fichiers\n(path, dirs, train_files) = next(os.walk(\"/kaggle/working/train\", topdown=True)) #utiliser next pour acceder au premier element\n(path, dirs, test_files) = next(os.walk(\"/kaggle/working/test1\", topdown=True))\n\ntrainSet_count = len(train_files)\ntestSet_count = len(test_files)\n\nprint(\"nombre d'image du trainSet: {}\".format(trainSet_count))\nprint(\"nombre d'image du testSet: {}\".format(testSet_count))","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:22:48.687031Z","iopub.execute_input":"2023-09-02T11:22:48.687657Z","iopub.status.idle":"2023-09-02T11:22:48.771634Z","shell.execute_reply.started":"2023-09-02T11:22:48.687624Z","shell.execute_reply":"2023-09-02T11:22:48.770717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### création des labels et du DataFrame","metadata":{}},{"cell_type":"code","source":"labels = [filename.split(\".\")[0] for filename in train_files] #liste de touts les labels du trainSet\n\ndata_train = pd.DataFrame({\"filename\": train_files, \"label\": labels})\ndata_test = pd.DataFrame({\"filename\": test_files})\n\ndata_train = data_train.sample(frac=1)\ndata_test = data_test.sample(frac=1)\nprint(data_train.head())\nprint(data_test.head())","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:22:51.354481Z","iopub.execute_input":"2023-09-02T11:22:51.354852Z","iopub.status.idle":"2023-09-02T11:22:51.392160Z","shell.execute_reply.started":"2023-09-02T11:22:51.354822Z","shell.execute_reply":"2023-09-02T11:22:51.391133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### regarder si il existe un déséquilibre entre les classes","metadata":{}},{"cell_type":"code","source":"data_train[\"label\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:22:54.153658Z","iopub.execute_input":"2023-09-02T11:22:54.154129Z","iopub.status.idle":"2023-09-02T11:22:54.211463Z","shell.execute_reply.started":"2023-09-02T11:22:54.154092Z","shell.execute_reply":"2023-09-02T11:22:54.210328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### affichage d'images","metadata":{}},{"cell_type":"code","source":"nb_images = 5\nrandList = random.choices(range(0,trainSet_count), k=nb_images)\n\nplt.figure(figsize=(15,10))\nfor i in range(1,nb_images+1):\n    \n    plt.subplot(4, 5, i)\n    \n    img = mpimg.imread(\"/kaggle/working/train/%s\" % train_files[randList[i-1]])\n    plt.imshow(img)\n    plt.title(\"label: \" + data_train[\"label\"][randList[i-1]])\n\nplt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-09-02T11:22:56.617025Z","iopub.execute_input":"2023-09-02T11:22:56.618409Z","iopub.status.idle":"2023-09-02T11:22:57.972222Z","shell.execute_reply.started":"2023-09-02T11:22:56.618367Z","shell.execute_reply":"2023-09-02T11:22:57.971226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Trouver la taille moyenne des images","metadata":{}},{"cell_type":"code","source":"# trouver la taille moyenne des images pour la génération de nouvelles données\nimage_paths = [\"/kaggle/working/train/%s\" % train_file for train_file in train_files]\n\nimages_height = []\nimages_width = []\n\nfor image_path in tqdm(image_paths, desc=\"image readed : \"):\n    \n    image = cv2.imread(image_path)\n    height, width, channel = image.shape\n\n    images_height.append(height)\n    images_width.append(width)\n\nmedian_height = np.median(images_height)\nmedian_width = np.median(images_width)\n\nprint(\"dimensions moyennes:\", median_width, \"x\", median_height)\n\n#les cnn travaillent avec des images carrées\nsize_square = min(int(median_height), int(median_width))//2 + 1\nprint(\"taille des nouvelles images générées: {}x{}\".format(size_square,size_square))\n","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:23:00.031537Z","iopub.execute_input":"2023-09-02T11:23:00.031919Z","iopub.status.idle":"2023-09-02T11:23:44.532304Z","shell.execute_reply.started":"2023-09-02T11:23:00.031888Z","shell.execute_reply":"2023-09-02T11:23:44.531201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Création du set de validation","metadata":{}},{"cell_type":"code","source":"labels = data_train[\"label\"]\n\ntrain_df, val_df = train_test_split(data_train, test_size=0.20, stratify=labels, random_state=42)\n\nprint(\"train_df: \", train_df.shape, train_df[\"label\"].value_counts(), \"\\n\" )\nprint(\"val_df: \", val_df.shape, val_df[\"label\"].value_counts())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-09-02T11:25:30.651435Z","iopub.execute_input":"2023-09-02T11:25:30.651887Z","iopub.status.idle":"2023-09-02T11:25:30.824075Z","shell.execute_reply.started":"2023-09-02T11:25:30.651853Z","shell.execute_reply":"2023-09-02T11:25:30.820934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Images par générateur","metadata":{}},{"cell_type":"markdown","source":"### création des images","metadata":{}},{"cell_type":"code","source":"batchSize = 128\n\ngenerator = ImageDataGenerator( \n    rotation_range=20, \n    width_shift_range=0.15,\n    height_shift_range=0.15,\n    rescale=1.0/255.0,\n    shear_range=0.2,\n    zoom_range=0.15,\n    horizontal_flip=True\n)\n\ntrain_idg = generator.flow_from_dataframe(\n    train_df,\n    directory= \"/kaggle/working/train/\",\n    x_col= \"filename\",\n    y_col= \"label\",\n    target_size= (size_square,size_square),\n    batch_size=batchSize,\n    class_mode=\"categorical\"\n)\nval_idg = generator.flow_from_dataframe(\n    val_df, \n    directory= \"/kaggle/working/train/\",\n    x_col= \"filename\",\n    y_col= \"label\",\n    target_size= (size_square,size_square),\n    batch_size=batchSize,\n    shuffle=False,\n    class_mode=\"categorical\"\n)\n\ngenerator_test = ImageDataGenerator(rescale=1.0/255.0)\nbatch_size_test = 125\n\ntest_idg = generator_test.flow_from_dataframe(\n    data_test, \n    directory= \"/kaggle/working/test1/\",\n    x_col= \"filename\",\n    y_col=None, #*\n    class_mode=None, #*\n    target_size= (size_square,size_square),\n    batch_size=batch_size_test, #doit être un multiple de 12500\n    shuffle=False #*\n)\n# *indispensable pour faire fonctionner le générateur sur un dataset sans labels (testset)\n'''Depreciated:\nIl ne faut plus utiliser ImageDataGenerator...\nplutôt tf.keras.utils.image_dataset_from_directory et transformer la sortie tf.data.Dataset avec \ndes couches de pré-traitement pour générer des images\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:25:34.397456Z","iopub.execute_input":"2023-09-02T11:25:34.397850Z","iopub.status.idle":"2023-09-02T11:25:34.838343Z","shell.execute_reply.started":"2023-09-02T11:25:34.397819Z","shell.execute_reply":"2023-09-02T11:25:34.837448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### affichage des images du générateur","metadata":{}},{"cell_type":"code","source":"example_df = train_df.sample(n=1)\nprint(example_df)\n\nexample_gen = generator.flow_from_dataframe(\n    example_df, \n    directory= \"/kaggle/working/train\", \n    x_col='filename',\n    y_col='label',\n    target_size=(size_square,size_square),\n    class_mode='categorical'\n)\n\nplt.figure(figsize=(12,12))\ni = 1\n\nfor batch, label in example_gen:\n    plt.subplot(4,5,i)\n    plt.imshow(batch[0])\n    i+=1\n    if i > 10:\n        break\nplt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-09-02T11:25:37.052360Z","iopub.execute_input":"2023-09-02T11:25:37.052741Z","iopub.status.idle":"2023-09-02T11:25:38.389060Z","shell.execute_reply.started":"2023-09-02T11:25:37.052686Z","shell.execute_reply":"2023-09-02T11:25:38.387960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## créer des callbacks","metadata":{}},{"cell_type":"code","source":"# en premier on créer un checkpoint, qui va garder seulement la meilleure époque en fonction du loss du valSet\nmodelCheckpoint = callbacks.ModelCheckpoint(\n    filepath=\"/kaggle/working/modelCheckpoint/\",\n    monitor=\"val_loss\",\n    save_best_only=True,\n    verbose=0\n)\n# ensuite réduire le learning rate quand notre accuracy du valSet n'augmente pas sur 3 étapes\nlearning_reduction = callbacks.ReduceLROnPlateau(\n    monitor=\"val_accuracy\",\n    patience=3,\n    factor=0.5,\n    min_lr=0.00001\n)\nprint(modelCheckpoint.filepath)","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:25:43.315649Z","iopub.execute_input":"2023-09-02T11:25:43.316034Z","iopub.status.idle":"2023-09-02T11:25:43.322324Z","shell.execute_reply.started":"2023-09-02T11:25:43.316006Z","shell.execute_reply":"2023-09-02T11:25:43.321333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Construction du model","metadata":{}},{"cell_type":"markdown","source":"### création du modèle","metadata":{}},{"cell_type":"code","source":"# on utilise un réseau convolutif, bcp plus efficace pour la classification d'images\nmodel = Sequential([\n    layers.Conv2D(32, (4,4), activation=\"relu\", input_shape=(size_square, size_square,3)),\n    layers.BatchNormalization(),\n    layers.MaxPool2D((2,2)),\n\n    layers.Conv2D(64, (4,4), activation=\"relu\"),\n    layers.BatchNormalization(),\n    layers.MaxPool2D((2,2)),\n    layers.Dropout(0.25),\n\n    layers.Conv2D(128, (4,4), activation=\"relu\"),\n    layers.BatchNormalization(),\n    layers.MaxPool2D((2,2)),\n    layers.Dropout(0.25),\n\n    layers.Flatten(),\n\n    layers.Dense(512 , activation=\"relu\"),\n    layers.BatchNormalization(),\n    layers.Dropout(0.25),\n\n    layers.Dense(2, activation=\"softmax\")\n    \n])\n\nmodel.compile(\n    loss=CategoricalCrossentropy(), \n    optimizer='adam',\n    metrics=['accuracy']\n)\n#https://keras.io/api/losses/\n#https://keras.io/api/optimizers/\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:25:46.221616Z","iopub.execute_input":"2023-09-02T11:25:46.222784Z","iopub.status.idle":"2023-09-02T11:25:49.575735Z","shell.execute_reply.started":"2023-09-02T11:25:46.222737Z","shell.execute_reply":"2023-09-02T11:25:49.574969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Entraînement du modèle","metadata":{}},{"cell_type":"code","source":"nbEpochs = 50 #40 suffisent avant overfitting\n\nhistory = model.fit(\n    train_idg,\n    epochs=nbEpochs,\n    validation_data=val_idg,\n    steps_per_epoch=train_df.shape[0] // batchSize,\n    validation_steps= val_df.shape[0] // batchSize, #double diviser pour cast en int\n    callbacks = [modelCheckpoint,learning_reduction]\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-02T11:25:54.607816Z","iopub.execute_input":"2023-09-02T11:25:54.608160Z","iopub.status.idle":"2023-09-02T15:11:56.634257Z","shell.execute_reply.started":"2023-09-02T11:25:54.608133Z","shell.execute_reply":"2023-09-02T15:11:56.633217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualisation des résultats du modèle","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,4))\n\nplt.subplot(1,2,1)\nplt.plot(history.history[\"loss\"], label=\"train_loss\",c=\"red\")\nplt.plot(history.history[\"val_loss\"], label=\"val_loss\",c=\"blue\")\nplt.title(\"loss\")\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(history.history[\"accuracy\"], label=\"train_acc\",c=\"red\")\nplt.plot(history.history[\"val_accuracy\"], label=\"val_acc\",c=\"blue\")\nplt.title(\"accuracy\")\nplt.legend()\n\nplt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-09-02T15:28:44.229013Z","iopub.execute_input":"2023-09-02T15:28:44.229389Z","iopub.status.idle":"2023-09-02T15:28:44.694637Z","shell.execute_reply.started":"2023-09-02T15:28:44.229359Z","shell.execute_reply":"2023-09-02T15:28:44.693687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rendu du fichier CSV","metadata":{}},{"cell_type":"markdown","source":"### effectuer les prédiction sur le TestSet","metadata":{}},{"cell_type":"code","source":"prediction = model.predict(\n    test_idg, #on utilise le générateur qui va redimensionner et normaliser notre dataSet\n    batch_size= batch_size_test,\n    steps= data_test.shape[0] // batch_size_test,\n    verbose=0\n)\nprint(prediction)","metadata":{"execution":{"iopub.status.busy":"2023-09-02T15:29:29.251311Z","iopub.execute_input":"2023-09-02T15:29:29.251683Z","iopub.status.idle":"2023-09-02T15:29:59.592267Z","shell.execute_reply.started":"2023-09-02T15:29:29.251654Z","shell.execute_reply":"2023-09-02T15:29:59.591309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rendre le csv","metadata":{}},{"cell_type":"code","source":"data_submission = pd.DataFrame({\n    \"id\":data_test[\"filename\"].str.split(\".\").str[0],\n    \"label\": np.argmax(prediction, axis=1),\n})\n\ndata_submission.to_csv(\"/kaggle/working/submission.csv\", index=False)\nprint(data_submission)","metadata":{"execution":{"iopub.status.busy":"2023-09-02T15:29:59.594217Z","iopub.execute_input":"2023-09-02T15:29:59.594573Z","iopub.status.idle":"2023-09-02T15:29:59.655350Z","shell.execute_reply.started":"2023-09-02T15:29:59.594538Z","shell.execute_reply":"2023-09-02T15:29:59.654414Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}