{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, I will be training a baseline CNN model for classifying the CIFAR 10 images.\n\n**Note**: Extracting the images from the 7z files is time consuming, especially for the images in the test set. In a separate [notebook](https://www.kaggle.com/code/syerramilli/cifar10-extract-images-as-numpy-arrays), I extracted the images into compressed `.npz` formats. I will be using this format instead.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom pathlib import Path\nfrom typing import Dict,Optional, List, Tuple\n\nplt.style.use('ggplot')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:25.485169Z","iopub.execute_input":"2023-09-05T18:31:25.485534Z","iopub.status.idle":"2023-09-05T18:31:34.144457Z","shell.execute_reply.started":"2023-09-05T18:31:25.485502Z","shell.execute_reply":"2023-09-05T18:31:34.143484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_folder = Path('/kaggle/input/cifar10-extract-images-as-numpy-arrays/')\ninput_folder = Path('/kaggle/input/cifar-10')\n\nIMG_HEIGHT = 32\nIMG_WIDTH = 32\nN_CLASSES = 10","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:34.146380Z","iopub.execute_input":"2023-09-05T18:31:34.146923Z","iopub.status.idle":"2023-09-05T18:31:34.152671Z","shell.execute_reply.started":"2023-09-05T18:31:34.146893Z","shell.execute_reply":"2023-09-05T18:31:34.151021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load training data and create label encoder","metadata":{}},{"cell_type":"code","source":"# load labels\nlabels_dat = pd.read_csv(input_folder/'trainLabels.csv',index_col=['id']).iloc[:,0]\n\n# build encoder from labels\nlabel_encoder = LabelEncoder().fit(labels_dat)\nclasses = label_encoder.classes_\nprint('Classes in CIFAR10')\nprint(classes)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:34.154495Z","iopub.execute_input":"2023-09-05T18:31:34.154902Z","iopub.status.idle":"2023-09-05T18:31:34.209985Z","shell.execute_reply.started":"2023-09-05T18:31:34.154840Z","shell.execute_reply":"2023-09-05T18:31:34.209013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_from_npz(filepath:Path) -> Tuple[np.ndarray]:\n    npz_out = np.load(filepath)\n    return npz_out['images'],npz_out['ids']","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:34.212529Z","iopub.execute_input":"2023-09-05T18:31:34.213333Z","iopub.status.idle":"2023-09-05T18:31:34.217900Z","shell.execute_reply.started":"2023-09-05T18:31:34.213300Z","shell.execute_reply":"2023-09-05T18:31:34.217005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load images \nimages, ids = load_data_from_npz(images_folder/'train.npz')\n\n# get corresponding labels\nlabels = label_encoder.transform(labels_dat.loc[ids])","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:34.219386Z","iopub.execute_input":"2023-09-05T18:31:34.220028Z","iopub.status.idle":"2023-09-05T18:31:38.214389Z","shell.execute_reply.started":"2023-09-05T18:31:34.219997Z","shell.execute_reply":"2023-09-05T18:31:38.213420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization utitities","metadata":{}},{"cell_type":"code","source":"def display_images(images:np.ndarray, labels:np.ndarray, title:str,seed:Optional[int]=None):\n\n    n = 10\n    \n    if seed is not None:\n        np.random.seed(seed)\n    idxs_to_show = np.random.choice(len(labels), size=n)\n    \n    imgs_show = images[idxs_to_show,...]\n    labels_show = labels[idxs_to_show]\n\n    fig = plt.figure(figsize=(20, 4))\n    plt.title(title)\n    plt.yticks([])\n    plt.xticks([])\n    \n    for i in range(10):\n        ax = fig.add_subplot(1, 10, i+1)\n        plt.xlabel(classes[labels_show[i]])\n        plt.xticks([])\n        plt.yticks([])\n        plt.imshow(imgs_show[i])\n        \n    return fig","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:38.216725Z","iopub.execute_input":"2023-09-05T18:31:38.218211Z","iopub.status.idle":"2023-09-05T18:31:38.227129Z","shell.execute_reply.started":"2023-09-05T18:31:38.218175Z","shell.execute_reply":"2023-09-05T18:31:38.225432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = display_images(images,labels,'Training data',seed=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:38.228625Z","iopub.execute_input":"2023-09-05T18:31:38.228970Z","iopub.status.idle":"2023-09-05T18:31:38.834342Z","shell.execute_reply.started":"2023-09-05T18:31:38.228923Z","shell.execute_reply":"2023-09-05T18:31:38.833394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN Architecture\nThe base architecture consists of one or more convolution layers followed by a dense layer with softmax activation. Optionally, there can be one or two dense layers with the ReLU activation between the convolution and the final softmax layer.\n\nThe function `construct_model` returns a compliled model with the architecture specified by the argument params. The function `build_and_fit` returns a model trained on the supplied data along with the optimization history.\n\n**Note**: The model expects input pixels to be in range [0,255]. The rescaling of these pixels to the unit interval [0,1] is done within the model itself through the `Rescaling` layer in Keras. ","metadata":{}},{"cell_type":"code","source":"def construct_model(params:Dict={}) -> tf.keras.Model:\n    '''\n    Builds a CNN based classification model with the specified hyperparameters\n    \n    Arguments\n    ------------\n    params: dict\n        Dictionary containing the hyperparameters of the model including the learning\n        rates\n        \n    Returns\n    -----------\n    model: tf.keras.Model\n        A compliled keras model\n    '''\n    \n    # initialize model\n    model = tf.keras.models.Sequential([\n        tf.keras.Input(shape=(IMG_WIDTH,IMG_HEIGHT,3)),\n        # preprocessing layers\n        tf.keras.layers.Rescaling(scale=1./255)\n    ])\n    \n    # image augmentation (optional)\n    # might help in improving the generalization performance \n    # default: no augmenting\n    if params.get('augment_images',0):\n        augmenting_layers = tf.keras.models.Sequential([\n            tf.keras.layers.RandomRotation(factor=0.1, interpolation='nearest'),\n            tf.keras.layers.RandomTranslation(\n                height_factor=0.1, width_fctor=0.1, interpolation='nearest'\n            )\n        ])\n        \n        model.add(augmenting_layers)\n        \n    # convolution blocks (default: 3) \n    # each block consists of a convolution layer followed by a \n    for i in range(params.get('n_conv_layers',3)):\n        model.add(tf.keras.layers.Conv2D(\n            params.get(f'n_filters_conv_{i+1}',16*2**i),kernel_size=(3,3),padding='same'\n        ))\n        model.add(tf.keras.layers.MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n    \n    # global average pooling\n    model.add(tf.keras.layers.GlobalAveragePooling2D())\n    \n    # optional feedforward layers before the softmax layer (default:1)\n    # range: 0-2\n    for i in range(params.get('n_dense_layers',1)):\n        model.add(tf.keras.layers.Dense(params.get(f'n_dense_units',128)))\n        model.add(tf.keras.layers.BatchNormalization())\n        model.add(tf.keras.layers.ReLU())\n        \n    # softmax\n    model.add(tf.keras.layers.Dense(N_CLASSES,activation='softmax'))\n    \n    \n    # compile\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=params.get('learning_rate',1e-3)),\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['accuracy']\n    )\n    \n    return model\n        \n\ndef build_and_fit(\n    params:Dict,X:np.ndarray,y:np.ndarray,\n    validation_split: float=0.1,\n    verbose:bool=False,\n    resample_seed:Optional[int]=None\n):\n    '''\n    Build and train a CNN model on the data\n    \n    Arguments\n    ---------\n    params: dict\n        Dictionary containing the hyperparameters of the model including learning rates\n        \n    X: np.ndarray\n        Pixels for the images with shape (batch_size,28,28,1)\n        \n    y: np.ndarray\n        Labels for the images with shape (batch_size,1)\n        \n    validation_split: float\n        Fraction of observations to be set aside as validation data (default: 0.1)\n        \n    verbose: bool\n        Whether or not to output training progress (Default: False)\n        \n    resample_seed: int, optional\n        Random Seed used for generating a validation split that will be used for early \n        stopping\n        \n    '''\n    \n    tf.keras.backend.clear_session()\n    \n    #further divide into train and validation splits for early stopping\n    # stratified on the labels\n    X_train,X_val,y_train,y_val = train_test_split(\n        X,y,test_size=validation_split,random_state=resample_seed,\n        stratify = y\n    )\n    \n    EPOCHS = 100 # maximum number of epochs\n    BATCH_SIZE = params.get('batch_size',128)\n    \n    # callbacks - reduce lr on plateau and early stopping\n    early_stopping = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        mode='min',\n        verbose=verbose,\n        patience=30,\n        restore_best_weights=True\n    )\n    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss',\n        mode='min',\n        verbose=verbose,\n        patience=10,\n        factor=0.5,\n        min_lr = 1e-5\n    )\n        \n    # create model\n    model = construct_model(params)\n\n    # train model\n    history = model.fit(\n        X_train,\n        y_train,\n        batch_size=BATCH_SIZE,\n        epochs=EPOCHS,\n        callbacks=[early_stopping,reduce_lr],\n        validation_data=(X_val, y_val),\n        verbose=verbose\n    )\n\n    return model,history","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:38.835446Z","iopub.execute_input":"2023-09-05T18:31:38.835771Z","iopub.status.idle":"2023-09-05T18:31:38.855698Z","shell.execute_reply.started":"2023-09-05T18:31:38.835741Z","shell.execute_reply":"2023-09-05T18:31:38.854816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train model","metadata":{}},{"cell_type":"code","source":"model,history = build_and_fit(\n    params={},X=images,y=labels,validation_split=0.1,resample_seed=1,verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:31:38.858799Z","iopub.execute_input":"2023-09-05T18:31:38.859405Z","iopub.status.idle":"2023-09-05T18:34:02.310708Z","shell.execute_reply.started":"2023-09-05T18:31:38.859373Z","shell.execute_reply":"2023-09-05T18:34:02.309812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save model\nmodel.save('cifar10_baseline_cnn.h5')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:34:02.314106Z","iopub.execute_input":"2023-09-05T18:34:02.314397Z","iopub.status.idle":"2023-09-05T18:34:02.359140Z","shell.execute_reply.started":"2023-09-05T18:34:02.314372Z","shell.execute_reply":"2023-09-05T18:34:02.358247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot training history\nmetrics = ['loss','accuracy']\nfig,axs = plt.subplots(1,len(metrics),figsize=(5*len(metrics),3.5),dpi=150)\n\nfor i,metric in enumerate(metrics):\n    \n    _ = axs[i].plot(history.history[f'{metric}'],label='Training')\n    _ = axs[i].plot(history.history[f'val_{metric}'],label='Validation')\n    _ = axs[i].legend()\n    _ = axs[i].set_ylabel(metric.capitalize())\n    _ = axs[i].set_xlabel('Epochs')\n\nfig.tight_layout()\nfig.savefig('cifar10_baseline_cnn_history.png',bbox_inches='tight')\nfig.savefig('cifar10_baseline_cnn_history.pdf',bbox_inches='tight')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:40:07.257500Z","iopub.execute_input":"2023-09-05T18:40:07.257857Z","iopub.status.idle":"2023-09-05T18:40:08.928785Z","shell.execute_reply.started":"2023-09-05T18:40:07.257826Z","shell.execute_reply":"2023-09-05T18:40:08.927871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test predictions","metadata":{}},{"cell_type":"code","source":"# load the test set images\ntest_images, test_ids = load_data_from_npz(images_folder/'test.npz')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:34:02.360306Z","iopub.execute_input":"2023-09-05T18:34:02.360632Z","iopub.status.idle":"2023-09-05T18:34:17.250882Z","shell.execute_reply.started":"2023-09-05T18:34:02.360601Z","shell.execute_reply":"2023-09-05T18:34:17.249848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get prediction probabilities\ntest_pred_probs = model.predict(test_images,batch_size=2000)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:34:17.252360Z","iopub.execute_input":"2023-09-05T18:34:17.252692Z","iopub.status.idle":"2023-09-05T18:34:20.396657Z","shell.execute_reply.started":"2023-09-05T18:34:17.252660Z","shell.execute_reply":"2023-09-05T18:34:20.395680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get prediction labels \ntest_preds = test_pred_probs.argmax(axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:34:20.398146Z","iopub.execute_input":"2023-09-05T18:34:20.398535Z","iopub.status.idle":"2023-09-05T18:34:20.415743Z","shell.execute_reply.started":"2023-09-05T18:34:20.398501Z","shell.execute_reply":"2023-09-05T18:34:20.414882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualizaing some of the test set predictions\n_ = display_images(test_images,test_preds,'Test data predictions',seed=0)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:40:13.059624Z","iopub.execute_input":"2023-09-05T18:40:13.060283Z","iopub.status.idle":"2023-09-05T18:40:13.624331Z","shell.execute_reply.started":"2023-09-05T18:40:13.060249Z","shell.execute_reply":"2023-09-05T18:40:13.623006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creating submission file\nsubmission = pd.DataFrame({\n    'id': test_ids,\n    'label':label_encoder.inverse_transform(test_preds)\n})\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T18:34:21.019373Z","iopub.execute_input":"2023-09-05T18:34:21.020028Z","iopub.status.idle":"2023-09-05T18:34:21.651545Z","shell.execute_reply.started":"2023-09-05T18:34:21.019994Z","shell.execute_reply":"2023-09-05T18:34:21.650570Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}