{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CIFAR 10 Image Classification\n\nMost of the code cells below include comments explaining the task to be performed in those cells. Please delete the comments and add code to perform those tasks. There are a few code cells in which code has already been provided for you. In some cases, you will need to complete this code.\n\n⚠ **NOTE:** You should make use of GPU acceleration in this notebook. \n\n","metadata":{}},{"cell_type":"markdown","source":"# Import Packages","metadata":{}},{"cell_type":"code","source":"# Import numpy, pandas, and matplotlib using the standard aliases. \n# Import mpimg from matplotlib.image\n# Import train_test_split from sklearn\n# Import pickle. \n# Import tensorflow and all needed tools from tensorflow.keras. \n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom sklearn.model_selection import train_test_split\n\n\nimport pickle\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-14T23:21:10.541572Z","iopub.execute_input":"2022-02-14T23:21:10.541822Z","iopub.status.idle":"2022-02-14T23:21:16.586696Z","shell.execute_reply.started":"2022-02-14T23:21:10.541748Z","shell.execute_reply":"2022-02-14T23:21:16.585955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Training DataFrame","metadata":{}},{"cell_type":"code","source":"# Load the training data into a DataFrame named 'train'. \n# Print the shape of the resulting DataFrame. \n# You do not need the test data in this notebook. \ntrain = pd.read_csv(f'../input/mu-cifar10/train.csv')\n\nprint('Training Set Size:', train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:16.588303Z","iopub.execute_input":"2022-02-14T23:21:16.58858Z","iopub.status.idle":"2022-02-14T23:21:16.640252Z","shell.execute_reply.started":"2022-02-14T23:21:16.588545Z","shell.execute_reply":"2022-02-14T23:21:16.639575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the head of the train DataFrame. \ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:16.641559Z","iopub.execute_input":"2022-02-14T23:21:16.641814Z","iopub.status.idle":"2022-02-14T23:21:16.658186Z","shell.execute_reply.started":"2022-02-14T23:21:16.64178Z","shell.execute_reply":"2022-02-14T23:21:16.657539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert 'label' to string\ntrain['label'] = train['label'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:16.660308Z","iopub.execute_input":"2022-02-14T23:21:16.660561Z","iopub.status.idle":"2022-02-14T23:21:16.714141Z","shell.execute_reply.started":"2022-02-14T23:21:16.660528Z","shell.execute_reply":"2022-02-14T23:21:16.713277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Distribution","metadata":{}},{"cell_type":"code","source":"# Display a DataFrame showing the proportion of observations with each \n# possible of the target variable (which is label). \n(train.label.value_counts() / len(train)).to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:16.715563Z","iopub.execute_input":"2022-02-14T23:21:16.715801Z","iopub.status.idle":"2022-02-14T23:21:16.738448Z","shell.execute_reply.started":"2022-02-14T23:21:16.715777Z","shell.execute_reply":"2022-02-14T23:21:16.737794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View Sample of Images","metadata":{}},{"cell_type":"code","source":"# Sample 16 images from the training set and display these along with their labels.\n# The images should be arranged in a 4x4 grid of subplots. \n# Please set the figure sizeto (6,6)\n\nsample = train.sample(n=16).reset_index()\n\nplt.figure(figsize=(6,6))\n\nfor i, row in sample.iterrows():\n\n    img = mpimg.imread(f'../input/mu-cifar10/train_images/{row.filename}')    \n    label = row.label\n\n    plt.subplot(4,4,i+1)\n    plt.imshow(img)\n    plt.text(0, -5, f'Class {label}', color='k')\n        \n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:16.739458Z","iopub.execute_input":"2022-02-14T23:21:16.739753Z","iopub.status.idle":"2022-02-14T23:21:17.521976Z","shell.execute_reply.started":"2022-02-14T23:21:16.739717Z","shell.execute_reply":"2022-02-14T23:21:17.521354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Generators","metadata":{}},{"cell_type":"code","source":"# Split the dataframe train into two DataFrames named train_df and valid_df. \n# Use 20% of the data for the validation set. \n# Use stratified sampling so that the label proportions are preserved.\n# Set a random seed for the split. \n\ntrain_df, valid_df = train_test_split(train, test_size=0.2, random_state=1, stratify=train.label)\n\nprint(train_df.shape)\nprint(valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:17.522888Z","iopub.execute_input":"2022-02-14T23:21:17.523117Z","iopub.status.idle":"2022-02-14T23:21:17.614244Z","shell.execute_reply.started":"2022-02-14T23:21:17.523088Z","shell.execute_reply":"2022-02-14T23:21:17.613441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create image data generators for both the training set and the validation set. \n# Use the data generators to scale the pixel values by a factor of 1/255. \n\ntrain_datagen = ImageDataGenerator(rescale=1/255)\nvalid_datagen = ImageDataGenerator(rescale=1/255)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:17.615701Z","iopub.execute_input":"2022-02-14T23:21:17.615973Z","iopub.status.idle":"2022-02-14T23:21:17.620175Z","shell.execute_reply.started":"2022-02-14T23:21:17.615939Z","shell.execute_reply":"2022-02-14T23:21:17.619488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"train.label = train.label.astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:17.621566Z","iopub.execute_input":"2022-02-14T23:21:17.622054Z","iopub.status.idle":"2022-02-14T23:21:17.630497Z","shell.execute_reply.started":"2022-02-14T23:21:17.62202Z","shell.execute_reply":"2022-02-14T23:21:17.629786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Complete the code for the data loaders below. \n\nBATCH_SIZE = 64\n\ntrain_loader = train_datagen.flow_from_dataframe(\n    dataframe = train_df,\n    directory = f'../input/mu-cifar10/train_images' ,\n    x_col = 'filename',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (32,32)\n)\n\nvalid_loader = train_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = f'../input/mu-cifar10/train_images',\n    x_col = 'filename',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = False,   # no need to shuffle validation set \n    class_mode = 'categorical',\n    target_size = (32,32)\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:21:17.634459Z","iopub.execute_input":"2022-02-14T23:21:17.634635Z","iopub.status.idle":"2022-02-14T23:24:05.564181Z","shell.execute_reply.started":"2022-02-14T23:21:17.634614Z","shell.execute_reply":"2022-02-14T23:24:05.563439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run this cell to determine the number of training and validation batches. \n\nTR_STEPS = len(train_loader)\nVA_STEPS = len(valid_loader)\n\nprint(TR_STEPS)\nprint(VA_STEPS)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:24:05.565492Z","iopub.execute_input":"2022-02-14T23:24:05.565923Z","iopub.status.idle":"2022-02-14T23:24:05.572516Z","shell.execute_reply.started":"2022-02-14T23:24:05.565871Z","shell.execute_reply":"2022-02-14T23:24:05.57165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Network","metadata":{}},{"cell_type":"code","source":"# Use this cell to construct a convolutional neural network model. \n# Your model should make use of each of the following layer types:\n#    Conv2D, MaxPooling2D, Dropout, BatchNormalization, Flatten, Dense\n# You can start by mimicking the architecture used in the \n# Aerial Cactus competetition, but you should explore different architectures\n# by adding more layers and/or adding more nodes in individual layers\n\nnp.random.seed(1)\ntf.random.set_seed(1)\n\ncnn = Sequential([\n    Conv2D(32, (3,3), activation = 'relu', padding = 'same', input_shape=(32,32,3)),\n    Conv2D(32, (3,3), activation = 'relu', padding = 'same'),\n    MaxPooling2D(2,2),\n    Dropout(0.5),\n    BatchNormalization(),\n\n    Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n    Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n    MaxPooling2D(2,2),\n    Dropout(0.5),\n    BatchNormalization(),\n\n    Flatten(),\n    \n    Dense(128, activation='relu'),\n    Dropout(0.5),\n    Dense(64, activation='relu'),\n    Dropout(0.5),\n    BatchNormalization(),\n    Dense(10, activation='softmax')   #10 values for the 'label'\n])\n\ncnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:24:05.574227Z","iopub.execute_input":"2022-02-14T23:24:05.574761Z","iopub.status.idle":"2022-02-14T23:24:08.208521Z","shell.execute_reply.started":"2022-02-14T23:24:05.574723Z","shell.execute_reply":"2022-02-14T23:24:08.207821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Network","metadata":{}},{"cell_type":"markdown","source":"# Training Run 1 ","metadata":{}},{"cell_type":"code","source":"# Define an optimizer and select a learning rate. \n# Then compile the model. \nopt = tf.keras.optimizers.Adam(0.001)\ncnn.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:24:08.210121Z","iopub.execute_input":"2022-02-14T23:24:08.210397Z","iopub.status.idle":"2022-02-14T23:24:08.230122Z","shell.execute_reply.started":"2022-02-14T23:24:08.210358Z","shell.execute_reply":"2022-02-14T23:24:08.22937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" Complete one or more training runs. \n Display training curves after each run.","metadata":{}},{"cell_type":"code","source":" %%time \n\nh1 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 20,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:24:08.232621Z","iopub.execute_input":"2022-02-14T23:24:08.233173Z","iopub.status.idle":"2022-02-14T23:42:01.553738Z","shell.execute_reply.started":"2022-02-14T23:24:08.233144Z","shell.execute_reply":"2022-02-14T23:42:01.552957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = h1.history\nprint(history.keys())","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:42:01.557099Z","iopub.execute_input":"2022-02-14T23:42:01.557399Z","iopub.status.idle":"2022-02-14T23:42:01.564688Z","shell.execute_reply.started":"2022-02-14T23:42:01.557371Z","shell.execute_reply":"2022-02-14T23:42:01.563659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:42:01.566279Z","iopub.execute_input":"2022-02-14T23:42:01.566566Z","iopub.status.idle":"2022-02-14T23:42:02.041058Z","shell.execute_reply.started":"2022-02-14T23:42:01.56653Z","shell.execute_reply":"2022-02-14T23:42:02.040372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Run 2","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.set_value(cnn.optimizer.learning_rate, 0.0001)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:42:02.042081Z","iopub.execute_input":"2022-02-14T23:42:02.04243Z","iopub.status.idle":"2022-02-14T23:42:02.048303Z","shell.execute_reply.started":"2022-02-14T23:42:02.042396Z","shell.execute_reply":"2022-02-14T23:42:02.047438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nh2 = cnn.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:42:02.049769Z","iopub.execute_input":"2022-02-14T23:42:02.050071Z","iopub.status.idle":"2022-02-15T00:01:51.675264Z","shell.execute_reply.started":"2022-02-14T23:42:02.050036Z","shell.execute_reply":"2022-02-15T00:01:51.674558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in history.keys():\n    history[k] += h2.history[k]\n\nepoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:01:51.678119Z","iopub.execute_input":"2022-02-15T00:01:51.679821Z","iopub.status.idle":"2022-02-15T00:01:52.138395Z","shell.execute_reply.started":"2022-02-15T00:01:51.67979Z","shell.execute_reply":"2022-02-15T00:01:52.137743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Model and History","metadata":{}},{"cell_type":"code","source":"# When you are satisfied with the model you have found, \n# save the model and the combined history dictionary to files.\n# Download these filesto your local device and then upload them \n# as a Kaggle dataset. \n\ncnn.save('cifarmodel2_v01.h5')\npickle.dump(history, open(f'cifarhistory2_v01.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:01:52.139603Z","iopub.execute_input":"2022-02-15T00:01:52.141044Z","iopub.status.idle":"2022-02-15T00:01:52.218936Z","shell.execute_reply.started":"2022-02-15T00:01:52.141004Z","shell.execute_reply":"2022-02-15T00:01:52.218249Z"},"trusted":true},"execution_count":null,"outputs":[]}]}