{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84511,"databundleVersionId":9468663,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RESNET aplicado a CIFAR-100","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:05.514021Z","iopub.execute_input":"2024-12-24T00:47:05.514437Z","iopub.status.idle":"2024-12-24T00:47:05.518157Z","shell.execute_reply.started":"2024-12-24T00:47:05.514398Z","shell.execute_reply":"2024-12-24T00:47:05.517403Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Carga de datos","metadata":{}},{"cell_type":"markdown","source":"### Train Test Split ","metadata":{}},{"cell_type":"code","source":"x_train = np.load(\"/kaggle/input/dl-itba-cifar-100-2024-q-1/x_train.npy\")/255.\nx_test = np.load(\"/kaggle/input/dl-itba-cifar-100-2024-q-1/x_test.npy\")/255.\ny_train_coarse = np.load(\"/kaggle/input/dl-itba-cifar-100-2024-q-1/y_train_coarse.npy\")\ny_train_fine = np.load(\"/kaggle/input/dl-itba-cifar-100-2024-q-1/y_train_fine.npy\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:05.519141Z","iopub.execute_input":"2024-12-24T00:47:05.519390Z","iopub.status.idle":"2024-12-24T00:47:06.035448Z","shell.execute_reply.started":"2024-12-24T00:47:05.519360Z","shell.execute_reply":"2024-12-24T00:47:06.034503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2024-12-24T00:47:06.036882Z","iopub.execute_input":"2024-12-24T00:47:06.037122Z","iopub.status.idle":"2024-12-24T00:47:06.040228Z","shell.execute_reply.started":"2024-12-24T00:47:06.037102Z","shell.execute_reply":"2024-12-24T00:47:06.039520Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle\nwith open(\"/kaggle/input/dl-itba-cifar-100-2024-q-1/fine_label_names.pck\", \"rb\") as f:\n    labels_fine = pickle.load(f)\nwith open(\"/kaggle/input/dl-itba-cifar-100-2024-q-1/coarse_label_names.pck\", \"rb\") as f:\n    labels_coarse = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T00:47:06.041332Z","iopub.execute_input":"2024-12-24T00:47:06.041621Z","iopub.status.idle":"2024-12-24T00:47:06.054724Z","shell.execute_reply.started":"2024-12-24T00:47:06.041572Z","shell.execute_reply":"2024-12-24T00:47:06.053974Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train,x_val, y_train, y_val = train_test_split(x_train, y_train_fine, test_size=0.1, random_state=42, stratify=y_train_fine)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T00:47:06.055517Z","iopub.execute_input":"2024-12-24T00:47:06.055829Z","iopub.status.idle":"2024-12-24T00:47:06.431985Z","shell.execute_reply.started":"2024-12-24T00:47:06.055802Z","shell.execute_reply":"2024-12-24T00:47:06.431286Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocesamiento\nSe realiza un preprocesamiento de las imágenes, en particular el proceso de *data augmentation* se aplica con el fin de aumentar la cantidad de imágenes en el conjunto train. Esto ya que el *data set* contiene pocas imagenes por clase. Se aplican técnicas como modificar el brillo, modificar la altura y ancho, aplicar rotaciones, etc.","metadata":{}},{"cell_type":"code","source":"# Agregamos ruido a las imágenes\nbrightness_factor = np.random.uniform(0.75, 1.25, (x_train.shape[0], 1,1,1))\nx_train = x_train * brightness_factor\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import array_to_img, img_to_array, load_img\n\n#Creamos un generador de imagenes\ndatagen = ImageDataGenerator(\n        width_shift_range=0.07,\n        height_shift_range=0.07,\n        shear_range=0.07,\n        zoom_range = 0.07,\n        horizontal_flip=True)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:06.432625Z","iopub.execute_input":"2024-12-24T00:47:06.432825Z","iopub.status.idle":"2024-12-24T00:47:06.825000Z","shell.execute_reply.started":"2024-12-24T00:47:06.432807Z","shell.execute_reply":"2024-12-24T00:47:06.824314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Definicion del modelo","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Flatten, Conv2D, MaxPooling2D, GlobalAveragePooling2D, Activation\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.regularizers import l1, l2\nfrom tensorflow.keras.constraints import MaxNorm\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, TensorBoard","metadata":{"execution":{"iopub.status.busy":"2024-12-24T00:47:06.825751Z","iopub.execute_input":"2024-12-24T00:47:06.825956Z","iopub.status.idle":"2024-12-24T00:47:06.830156Z","shell.execute_reply.started":"2024-12-24T00:47:06.825939Z","shell.execute_reply":"2024-12-24T00:47:06.829247Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Evolución de arquitecturas testeadas \nEn un principio se comenzó por realizar *fine tuning* a una red VGG con los pesos preentrenados de *Image Net*. En un comienzo se liberaron solo las ultimas 4 capas, luego se vieron mejores resultados liberando completamente los pesos. Para todos los entrenamientos se fue iterando con distintas herramientas de regularización, se utilizó L2, batch-normalization y dropout. El modelo final no superaba el 60% de accuracy en validation,llegando a 80% en train. \n\nDentro de este notebook se intenta evaluar si una red del tipo RES NET obtine mejores resultados al hacer uso de las *skip connections*.Se comenzó entrenando una RES NET 50 con todos sus pesos libres y con 3 capas densas adicionales con los métodos de regularización previmente menscionados. Los resultados eran mejores que en la VGG, con una *accuracy* en validación muy cercana al 60%. Sin embargo, se planteó la posibilidad de que el tamaño de la red para imágenes de tan poca dimensión sea un problema. Por ello en este notebook se crea una arquitectura RES NET de 20 capas. Se utiliza Batch-normalization y L2 dentro de las capas convolucionales como regularizadores. A su vez, se continua añadiendo 3 capas densas al final.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.initializers import HeNormal\nfrom tensorflow.keras.layers import Input, Conv2D, BatchNormalization, Activation, Add, GlobalAveragePooling2D, Dense, Flatten, Dropout","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:06.832617Z","iopub.execute_input":"2024-12-24T00:47:06.832852Z","iopub.status.idle":"2024-12-24T00:47:06.841824Z","shell.execute_reply.started":"2024-12-24T00:47:06.832824Z","shell.execute_reply":"2024-12-24T00:47:06.841166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resnet_block(inputs, filters, strides=1, kernel_initializer=\"he_normal\", l2_reg=0.001):\n    \"\"\"Construye un bloque básico de ResNet con inicialización y regularización configurables.\"\"\"\n    x = Conv2D(filters, kernel_size=3, strides=strides, padding=\"same\", \n               kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(inputs)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n\n    x = Conv2D(filters, kernel_size=3, strides=1, padding=\"same\", \n               kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(x)\n    x = BatchNormalization()(x)\n\n    if strides != 1 or inputs.shape[-1] != filters:\n        shortcut = Conv2D(filters, kernel_size=1, strides=strides, padding=\"same\", \n                          kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(inputs)\n        shortcut = BatchNormalization()(shortcut)\n    else:\n        shortcut = inputs\n\n    x = Add()([x, shortcut])\n    x = Activation(\"relu\")(x)\n    return x\n\ndef resnet_cifar_with_dense(input_shape, num_classes, depth=20, l2_reg=0.001, dropout_rate=0.1, kernel_initializer=\"he_normal\"):\n    \"\"\"Construye una ResNet para CIFAR con capas densas adicionales configurables.\"\"\"\n    num_blocks = (depth - 2) // 6  # Bloques por etapa\n    inputs = Input(shape=input_shape)\n    \n    # Primera capa\n    x = Conv2D(16, kernel_size=3, strides=1, padding=\"same\", \n               kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(inputs)\n    x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n\n    # Bloques de ResNet\n    for filters in [16, 32, 64]:  # Tres etapas con diferentes filtros\n        for block in range(num_blocks):\n            strides = 2 if block == 0 and filters != 16 else 1  # Downsampling en la primera capa de cada etapa\n            x = resnet_block(x, filters, strides, kernel_initializer, l2_reg)\n\n    # Capas densas adicionales\n    x = Flatten()(x)\n    x = Dense(1024, activation='relu', kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(dropout_rate)(x)\n    x = Dense(256, activation='relu', kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(dropout_rate)(x)\n    outputs = Dense(num_classes, activation='softmax', kernel_initializer=kernel_initializer, kernel_regularizer=l2(l2_reg))(x)\n\n    model = Model(inputs, outputs)\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:06.843213Z","iopub.execute_input":"2024-12-24T00:47:06.843460Z","iopub.status.idle":"2024-12-24T00:47:06.853750Z","shell.execute_reply.started":"2024-12-24T00:47:06.843435Z","shell.execute_reply":"2024-12-24T00:47:06.853058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow import distribute","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:06.854469Z","iopub.execute_input":"2024-12-24T00:47:06.854778Z","iopub.status.idle":"2024-12-24T00:47:06.868211Z","shell.execute_reply.started":"2024-12-24T00:47:06.854749Z","shell.execute_reply":"2024-12-24T00:47:06.867539Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Compilado del modelo\nSe compila el modelo utilizando una estartegia que divide la carga dentro de las 2 GPU que Kagle provee","metadata":{}},{"cell_type":"code","source":"strategy = distribute.MirroredStrategy()\nwith strategy.scope():\n    input_shape = (32, 32, 3)\n    num_classes = 100  # CIFAR-100\n    depth = 20\n    l2_reg = 0.001  # Regularización L2\n    dropout_rate = 0.2  # Dropout\n    kernel_initializer = HeNormal()  # Inicialización HeNormal\n\n    model = resnet_cifar_with_dense(input_shape, num_classes, depth, l2_reg, dropout_rate, kernel_initializer)\n    learning_rate = 0.001\n    optimizer = Adam(learning_rate=learning_rate)\n    model.compile(\n        optimizer=optimizer, \n        loss='sparse_categorical_crossentropy', \n        metrics=['accuracy']\n    )\n    model.build(input_shape=(None, 32, 32, 3))  # Explicit input shape definition\n    model.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:06.868959Z","iopub.execute_input":"2024-12-24T00:47:06.869231Z","iopub.status.idle":"2024-12-24T00:47:08.455965Z","shell.execute_reply.started":"2024-12-24T00:47:06.869205Z","shell.execute_reply":"2024-12-24T00:47:08.455185Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Entrenamiento\n\nPara entrenar el modelo, utilizamos los siguientes callbacks:\n\n- Reduce Learning Rate on Plateau, para regularizar.\n- Model checkpoint, para no perder el progreso parcial si por algún motivo se cerrase la sesión de entrenamiento.\n- Early stopping, para quedarnos siempre con el modelo que mejor ajustó a la métrica de interés en validation, y para evitar un sobreentrenamiento que derive en overfitting.","metadata":{}},{"cell_type":"code","source":"rlrop = ReduceLROnPlateau(\n    monitor = \"val_accuracy\",\n    factor = 0.5,\n    patience = 5,\n    verbose = 1,\n    min_lr = 1e-5\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:08.456678Z","iopub.execute_input":"2024-12-24T00:47:08.456891Z","iopub.status.idle":"2024-12-24T00:47:08.460486Z","shell.execute_reply.started":"2024-12-24T00:47:08.456873Z","shell.execute_reply":"2024-12-24T00:47:08.459664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mc = ModelCheckpoint(\n    \"best_weights.weights.h5\",\n    monitor = \"val_accuracy\",\n    verbose = 1,\n    save_best_only = True,\n    save_weights_only = True,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:08.461326Z","iopub.execute_input":"2024-12-24T00:47:08.461611Z","iopub.status.idle":"2024-12-24T00:47:08.472310Z","shell.execute_reply.started":"2024-12-24T00:47:08.461563Z","shell.execute_reply":"2024-12-24T00:47:08.471605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"es = EarlyStopping(\n    monitor = \"val_accuracy\",\n    patience = 8, \n    verbose = 1,\n    restore_best_weights = True,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:08.473062Z","iopub.execute_input":"2024-12-24T00:47:08.473303Z","iopub.status.idle":"2024-12-24T00:47:08.483644Z","shell.execute_reply.started":"2024-12-24T00:47:08.473273Z","shell.execute_reply":"2024-12-24T00:47:08.482887Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Recuperación de pesos previamente guardados, si los hay","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    datagen.flow(x_train, y_train, batch_size=128), # aplicamos el generador de imagenes\n    epochs=200, \n    validation_data = (x_val, y_val), \n    callbacks= [\n        rlrop,\n        es,\n        mc\n    ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T00:47:08.495602Z","iopub.execute_input":"2024-12-24T00:47:08.495898Z","iopub.status.idle":"2024-12-24T01:50:25.640296Z","shell.execute_reply.started":"2024-12-24T00:47:08.495878Z","shell.execute_reply":"2024-12-24T01:50:25.639643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.save_weights('model2_resnet20.weights.h5')\nmodel.load_weights('model2_resnet20.weights.h5')\nprint(\"Weights loaded successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T01:56:32.257850Z","iopub.execute_input":"2024-12-24T01:56:32.258164Z","iopub.status.idle":"2024-12-24T01:56:33.273332Z","shell.execute_reply.started":"2024-12-24T01:56:32.258144Z","shell.execute_reply":"2024-12-24T01:56:33.272611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.save('model2_resnet.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T01:56:34.287073Z","iopub.execute_input":"2024-12-24T01:56:34.287357Z","iopub.status.idle":"2024-12-24T01:56:34.641318Z","shell.execute_reply.started":"2024-12-24T01:56:34.287337Z","shell.execute_reply":"2024-12-24T01:56:34.640127Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluación de resultados","metadata":{}},{"cell_type":"markdown","source":"### Gráficas ","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T01:57:04.576508Z","iopub.execute_input":"2024-12-24T01:57:04.576770Z","iopub.status.idle":"2024-12-24T01:57:04.580097Z","shell.execute_reply.started":"2024-12-24T01:57:04.576749Z","shell.execute_reply":"2024-12-24T01:57:04.579239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot training and validation accuracy\nplt.plot(history.history[\"accuracy\"], label=\"Training Accuracy\")\nplt.plot(history.history[\"val_accuracy\"], label=\"Validation Accuracy\")\n\n# Add a title, labels, and legend\nplt.title(\"Model Accuracy\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.legend(loc=\"best\")\n\n# Show the plot\nplt.grid(False)  # Optional: Adds grid lines for better readability\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T01:57:05.776402Z","iopub.execute_input":"2024-12-24T01:57:05.776739Z","iopub.status.idle":"2024-12-24T01:57:06.060049Z","shell.execute_reply.started":"2024-12-24T01:57:05.776711Z","shell.execute_reply":"2024-12-24T01:57:06.059174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot training and validation accuracy\nplt.plot(history.history[\"loss\"], label=\"Training loss\")\nplt.plot(history.history[\"val_loss\"], label=\"Validation loss\")\n\n# Add a title, labels, and legend\nplt.title(\"Model Loss\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.legend(loc=\"best\")\n\n# Show the plot\nplt.grid(False)  # Optional: Adds grid lines for better readability\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T01:57:09.122977Z","iopub.execute_input":"2024-12-24T01:57:09.123285Z","iopub.status.idle":"2024-12-24T01:57:09.390218Z","shell.execute_reply.started":"2024-12-24T01:57:09.123262Z","shell.execute_reply":"2024-12-24T01:57:09.389328Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Matriz de confusión","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay\nfrom sklearn.metrics import confusion_matrix","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T01:57:22.460173Z","iopub.execute_input":"2024-12-24T01:57:22.460371Z","iopub.status.idle":"2024-12-24T01:57:22.463756Z","shell.execute_reply.started":"2024-12-24T01:57:22.460354Z","shell.execute_reply":"2024-12-24T01:57:22.462937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Unique labels\nunique_labels = np.unique(y_train_fine)\n\n# Predictions and confusion matrix\ny_pred = model.predict(x_val).argmax(axis=1)\ncm = confusion_matrix(y_val, y_pred)\n\n# Normalize the confusion matrix\ncm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n# Select only the first 10 rows and columns\ncm_subset = cm[:10, :10]\nlabels_subset = labels_fine[:10]\n\n# Increase figure size\nplt.figure(figsize=(40, 40))  # Adjust size for the smaller matrix\n\n# Create and show the confusion matrix for the subset\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_subset, display_labels=labels_subset)\ndisp.plot(cmap=plt.cm.Blues, values_format=\".2f\", xticks_rotation='vertical')\n\n# Adjust label size\nplt.xticks(fontsize=10)\nplt.yticks(fontsize=10)\n\n# Add title\nplt.title('Subset de la matriz de confusión para CIFAR-100', fontsize=16, fontweight=\"bold\")\n\n# Show plot\n#plt.tight_layout()\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T02:11:51.127760Z","iopub.execute_input":"2024-12-24T02:11:51.128069Z","iopub.status.idle":"2024-12-24T02:11:53.482125Z","shell.execute_reply.started":"2024-12-24T02:11:51.128048Z","shell.execute_reply":"2024-12-24T02:11:53.481364Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Conclusiones\nEvaluando los resultados, y considerando la evolución del modelo a lo largo de nuestras distintas iteraciones, concluimos que:\n\n- Las capas de batch-normalization ayudan a generalizar los resultados.\n- Los callbacks para guardado de checkpoints y early stopping permiten entrenar más modelos en menos tiempo, lo que optmiza la exploración de arquitecturas.\n- Los resultados superan con creces a los obtenidos con MLP.\n- Si bien la mejoría no es drástica el hecho de usar una Res Net de 20 capas da mejores resultados. Aludiendo a que el tamaño y las *skip connections* son cruciales.","metadata":{}}]}