{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"# List contents of the working directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-10T23:54:36.268856Z","iopub.execute_input":"2023-07-10T23:54:36.269295Z","iopub.status.idle":"2023-07-10T23:54:36.324818Z","shell.execute_reply.started":"2023-07-10T23:54:36.269253Z","shell.execute_reply":"2023-07-10T23:54:36.323391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Python package to open .7z archives\n!pip install py7zr;","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:54:36.327616Z","iopub.execute_input":"2023-07-10T23:54:36.329224Z","iopub.status.idle":"2023-07-10T23:54:51.962429Z","shell.execute_reply.started":"2023-07-10T23:54:36.329184Z","shell.execute_reply":"2023-07-10T23:54:51.961145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import libraries\n\n# Loading and processing\nimport glob\nimport cv2\nimport py7zr\n\n# Data manipulation\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# Tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:54:51.964351Z","iopub.execute_input":"2023-07-10T23:54:51.966998Z","iopub.status.idle":"2023-07-10T23:54:59.017333Z","shell.execute_reply.started":"2023-07-10T23:54:51.966950Z","shell.execute_reply":"2023-07-10T23:54:59.016134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:54:59.020575Z","iopub.execute_input":"2023-07-10T23:54:59.021467Z","iopub.status.idle":"2023-07-10T23:54:59.997660Z","shell.execute_reply.started":"2023-07-10T23:54:59.021418Z","shell.execute_reply":"2023-07-10T23:54:59.996372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look for GPU\ndevice_name = tf.test.gpu_device_name()\nif device_name != '/device:GPU:0':\n    print('GPU device not found')\nprint('Found GPU at: {}'.format(device_name))","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:54:59.999672Z","iopub.execute_input":"2023-07-10T23:55:00.000786Z","iopub.status.idle":"2023-07-10T23:55:02.049777Z","shell.execute_reply.started":"2023-07-10T23:55:00.000736Z","shell.execute_reply":"2023-07-10T23:55:02.048583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the data","metadata":{}},{"cell_type":"code","source":"# Class labels\nCLASS_NAMES = ['airplane', 'automobile', 'bird', 'cat', 'deer',\n               'dog', 'frog', 'horse', 'ship', 'truck']","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:55:02.051397Z","iopub.execute_input":"2023-07-10T23:55:02.053557Z","iopub.status.idle":"2023-07-10T23:55:02.063553Z","shell.execute_reply.started":"2023-07-10T23:55:02.053514Z","shell.execute_reply":"2023-07-10T23:55:02.062360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Open training archive file\n\nif not len(glob.glob('/kaggle/working/train/*.png')):\n    with py7zr.SevenZipFile('/kaggle/input/cifar-10/train.7z', mode='r') as z:\n        z.extractall()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:55:02.066137Z","iopub.execute_input":"2023-07-10T23:55:02.066450Z","iopub.status.idle":"2023-07-10T23:56:01.841871Z","shell.execute_reply.started":"2023-07-10T23:55:02.066421Z","shell.execute_reply":"2023-07-10T23:56:01.840684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the filepath to an integer image ID\ndef filename_to_id(filename):\n    return int(filename.split('/')[-1].split('.')[0])\n\n# Load all of the images into an array, also return the image IDs\ndef load_imgs_and_ids(img_directory_path):\n    image_filenames = glob.glob(img_directory_path + '/*.png')\n    list_of_images = []\n    image_ids = []\n    for f in image_filenames:\n        image_id = filename_to_id(f) # E.G. '/kaggle/working/train/44240.png' -> 44240\n        image_ids.append(image_id)\n        data_array = cv2.imread(f)\n        list_of_images.append(data_array)\n\n    X = np.array(list_of_images)\n    \n    return X, image_ids\n    \nX_train, train_img_ids = load_imgs_and_ids('/kaggle/working/train')\n\n# Read the labels from the CSV files and convert the array of IDs into a 0-9 code\ntrain_labels_df = pd.read_csv('/kaggle/input/cifar-10/trainLabels.csv')\nimg_codes = [CLASS_NAMES.index(train_labels_df.loc[img_id-1]['label']) for img_id in train_img_ids]\ny_train = np.array(img_codes)\n\nX_train.shape, y_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:01.843372Z","iopub.execute_input":"2023-07-10T23:56:01.844427Z","iopub.status.idle":"2023-07-10T23:56:13.408134Z","shell.execute_reply.started":"2023-07-10T23:56:01.844385Z","shell.execute_reply":"2023-07-10T23:56:13.406931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Alternatively, could use data from Tensorflow\n\"\"\"\nfrom tensorflow.keras.datasets import cifar10\n\n(X_train,y_train),(some_more_images,some_more_labels) = cifar10.load_data()\nprint(X_train.shape, y_train.shape)\n\ny_train = y_train.reshape((50000,))\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:13.409937Z","iopub.execute_input":"2023-07-10T23:56:13.410333Z","iopub.status.idle":"2023-07-10T23:56:13.417764Z","shell.execute_reply.started":"2023-07-10T23:56:13.410294Z","shell.execute_reply":"2023-07-10T23:56:13.416636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# RGB pixel values lie on a [0,...,255] scale. We rescale the channels to have channel intensities lie \n# from 0 to 1 inclusive\n\nX_train = X_train / 255.0","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:13.422533Z","iopub.execute_input":"2023-07-10T23:56:13.423258Z","iopub.status.idle":"2023-07-10T23:56:13.801730Z","shell.execute_reply.started":"2023-07-10T23:56:13.423220Z","shell.execute_reply":"2023-07-10T23:56:13.800605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"# To get a sense of the data we work with\n\nplt.rcParams[\"figure.figsize\"] = (10,10)\nfig,axes = plt.subplots(10,10)\nfor i,ax in enumerate(axes.flatten()):\n  ax.imshow(X_train[i])\n  ax.set_title(f'{CLASS_NAMES[y_train[i]]}')\n  ax.set_xticks([])\n  ax.set_yticks([])\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:13.803199Z","iopub.execute_input":"2023-07-10T23:56:13.804195Z","iopub.status.idle":"2023-07-10T23:56:18.674383Z","shell.execute_reply.started":"2023-07-10T23:56:13.804149Z","shell.execute_reply":"2023-07-10T23:56:18.673066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convolutional Neural Network Architecture","metadata":{}},{"cell_type":"code","source":"# CNN model, with 6 convolutional layers, 3 pooling layers, and 3 dense layers. Softmax output layer.\nmodel = models.Sequential()\n\nmodel.add(layers.Conv2D(64, (5,5), padding='same', activation='relu', input_shape=(32,32,3)))\nmodel.add(layers.Conv2D(128, (3,3), padding='same', activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2,2),strides=(2,2)))\n\nmodel.add(layers.Conv2D(128, (3,3), padding='same', activation='relu'))\nmodel.add(layers.Conv2D(128, (3,3), padding='same', activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(layers.Conv2D(128, (3,3), padding='same', activation='relu'))\nmodel.add(layers.Conv2D(128, (3,3), padding='same', activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(128, activation='relu'))\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:18.675486Z","iopub.execute_input":"2023-07-10T23:56:18.675840Z","iopub.status.idle":"2023-07-10T23:56:19.294124Z","shell.execute_reply.started":"2023-07-10T23:56:18.675805Z","shell.execute_reply":"2023-07-10T23:56:19.293287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:19.295435Z","iopub.execute_input":"2023-07-10T23:56:19.295938Z","iopub.status.idle":"2023-07-10T23:56:19.329066Z","shell.execute_reply.started":"2023-07-10T23:56:19.295897Z","shell.execute_reply":"2023-07-10T23:56:19.327963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmenting the training data and training the model","metadata":{}},{"cell_type":"code","source":"# We augment the training data within the mini-batches to try to reduce generalization error\ndatagen = ImageDataGenerator(\n    horizontal_flip=True,\n    channel_shift_range=0.05,\n    rotation_range=10,\n    validation_split=0.15\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:19.330414Z","iopub.execute_input":"2023-07-10T23:56:19.331434Z","iopub.status.idle":"2023-07-10T23:56:19.337202Z","shell.execute_reply.started":"2023-07-10T23:56:19.331395Z","shell.execute_reply":"2023-07-10T23:56:19.336045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# An example transformation\n\nimg = X_train[4]\n\nfig, ax = plt.subplots(1,2)\nax[0].imshow(img)\nax[0].set_title(\"Original image\")\nax[1].imshow(datagen.apply_transform(img,{'theta':-10,'channel_shift_intensity':0.05,'flip_horizontal':True}))\nax[1].set_title(\"Example transformation\")\n\nfor i in range(2):\n  ax[i].set_xticks([])\n  ax[i].set_yticks([])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:19.338575Z","iopub.execute_input":"2023-07-10T23:56:19.339939Z","iopub.status.idle":"2023-07-10T23:56:19.499126Z","shell.execute_reply.started":"2023-07-10T23:56:19.339899Z","shell.execute_reply":"2023-07-10T23:56:19.497886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model, including an early stopping if we can't reduce the loss further\n\nearly_stopping = keras.callbacks.EarlyStopping(monitor='loss',patience=10)\n\ntraining_flow = datagen.flow(X_train, y_train, subset='training')\nvalidation_flow = datagen.flow(X_train, y_train, subset='validation', shuffle=False)\n\nhistory = model.fit(training_flow,validation_split=0.1,\n                    verbose=1,\n                    validation_data=validation_flow,\n                    callbacks=[early_stopping],\n                    epochs=70)","metadata":{"execution":{"iopub.status.busy":"2023-07-10T23:56:19.501101Z","iopub.execute_input":"2023-07-10T23:56:19.501866Z","iopub.status.idle":"2023-07-11T00:44:28.428025Z","shell.execute_reply.started":"2023-07-10T23:56:19.501820Z","shell.execute_reply":"2023-07-11T00:44:28.426924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation","metadata":{}},{"cell_type":"code","source":"# Plot the evolution of training/validation error and accuracy\nevaluation = model.evaluate(X_train, y_train)\nprint(f'Test Accuracy : {evaluation[1] * 100:.2f}%')\n\ntrain_metrics = history.history\n\nfig, ax = plt.subplots(1,2)\nax[0].plot(train_metrics['loss'])\nax[0].plot(train_metrics['val_loss'])\nax[0].legend(['train','val'], loc='upper right')\nax[0].set_xlabel(\"Epoch\")\nax[0].set_ylabel(\"Loss\")\nax[0].set_title(\"Loss over the training epochs\")\n\nax[1].plot(train_metrics['accuracy'])\nax[1].plot(train_metrics['val_accuracy'])\nax[1].legend(['train','val'], loc='upper right')\nax[1].set_xlabel(\"Epoch\")\nax[1].set_ylabel(\"Accuracy\")\nax[1].set_title(\"Accuracy over the training epochs\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-11T00:46:53.456281Z","iopub.execute_input":"2023-07-11T00:46:53.457274Z","iopub.status.idle":"2023-07-11T00:47:02.093745Z","shell.execute_reply.started":"2023-07-11T00:46:53.457212Z","shell.execute_reply":"2023-07-11T00:47:02.092688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example predictions\n\nsample_set = X_train[100:125]\n\nsample_predictions = np.argmax(model.predict(sample_set),axis=1)\n\nfig,axes = plt.subplots(5,5)\nfor i,ax in enumerate(axes.flatten()):\n  ax.imshow(sample_set[i])\n  ax.set_title(f'{CLASS_NAMES[sample_predictions[i]]}')\n  ax.set_xticks([])\n  ax.set_yticks([])\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-07-11T00:52:11.728169Z","iopub.execute_input":"2023-07-11T00:52:11.728756Z","iopub.status.idle":"2023-07-11T00:52:13.481742Z","shell.execute_reply.started":"2023-07-11T00:52:11.728684Z","shell.execute_reply":"2023-07-11T00:52:13.480676Z"},"trusted":true},"execution_count":null,"outputs":[]}]}