{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**<center><span style= \"font-family: Verdana ; font-size:32px; color: black;\">Training a Convolutional neural network model on the Cifar10 dataset</span></center>**\n<span style = \"font-family: Cursive ;font-size:28px; \">[CIFAR-10](https://paperswithcode.com/dataset/cifar-10)</span>","metadata":{}},{"cell_type":"markdown","source":"## **Step 1**: Importing necessary Libraries to configure the enviroment","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nimport keras\nfrom keras.utils.np_utils import to_categorical # convert labels to one-hot-encoding\nimport matplotlib.pyplot as plt\n\n!pip install py7zr\nimport py7zr\nimport shutil\nfrom py7zr import unpack_7zarchive\n\nfrom PIL import Image\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation, Dropout\n\nfrom keras.layers.advanced_activations import LeakyReLU\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import BatchNormalization\nfrom keras import backend as K\nimport tensorflow_addons as tfa","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:37:42.863865Z","iopub.execute_input":"2022-09-20T12:37:42.864270Z","iopub.status.idle":"2022-09-20T12:38:06.414670Z","shell.execute_reply.started":"2022-09-20T12:37:42.864194Z","shell.execute_reply":"2022-09-20T12:38:06.413462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extracting train and test archives by py7zr library","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x ../input/cifar-10/train.7z\n#!python -m py7zr x ../input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:38:06.417221Z","iopub.execute_input":"2022-09-20T12:38:06.418332Z","iopub.status.idle":"2022-09-20T12:38:51.351548Z","shell.execute_reply.started":"2022-09-20T12:38:06.418286Z","shell.execute_reply":"2022-09-20T12:38:51.350076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading the given labels for the test and train data ","metadata":{}},{"cell_type":"code","source":"y_train = pd.read_csv('/kaggle/input/cifar-10/trainLabels.csv')\nY_test = pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv')\n\nY_train = np.array(y_train['label'])\n#Y_test = np.array(y_test['label'])\nprint(Y_train.shape)\nprint(Y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:38:51.353435Z","iopub.execute_input":"2022-09-20T12:38:51.353883Z","iopub.status.idle":"2022-09-20T12:38:51.503082Z","shell.execute_reply.started":"2022-09-20T12:38:51.353838Z","shell.execute_reply":"2022-09-20T12:38:51.502095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using Glob() function to find Image files recursively","metadata":{}},{"cell_type":"code","source":"filelist = glob.glob(\"train/*\")\nfilelist = sorted(filelist,key= lambda x:int(x.split(\"/\")[-1].split(\".\")[0]))\nx_train = np.array([np.array(Image.open(fname)) for fname in filelist])\nprint(f\"X.shape : {x_train.shape} y.shape : {y_train.shape}\")\n\nfilelist = glob.glob(\"test/*\")\nfilelist = sorted(filelist,key= lambda x:int(x.split(\"/\")[-1].split(\".\")[0]))\nx_test = np.array([np.array(Image.open(fname)) for fname in filelist])\nprint(f\"X.shape : {x_test.shape} y.shape : {Y_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:38:51.506081Z","iopub.execute_input":"2022-09-20T12:38:51.506407Z","iopub.status.idle":"2022-09-20T12:39:00.658484Z","shell.execute_reply.started":"2022-09-20T12:38:51.506380Z","shell.execute_reply":"2022-09-20T12:39:00.656573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalizing the train and test data and also converting the class labels to one-hot encoding","metadata":{}},{"cell_type":"code","source":"#normalize inputs:\n#X_train = x_train/255 #- 0.5\n#X_test = x_test/255 #- 0.5\nX_train = x_train\nX_test = x_test/255\n\ncifar10_classes = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\n\n# For the above class names, we need to convert these categorical data into numerical values.  \ny_train_indexes = []\nfor classs in Y_train:\n    y_train_indexes.append(cifar10_classes.index(classs))\n    \n# convert class labels to one-hot encoded:\nY_train = to_categorical(y_train_indexes)\nY_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:39:00.659889Z","iopub.execute_input":"2022-09-20T12:39:00.660444Z","iopub.status.idle":"2022-09-20T12:39:00.691433Z","shell.execute_reply.started":"2022-09-20T12:39:00.660408Z","shell.execute_reply":"2022-09-20T12:39:00.690406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Convolutional Neural Network  architecture","metadata":{}},{"cell_type":"code","source":"def make_model():\n    model = Sequential()\n    \n    model.add( Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same', input_shape=(32, 32, 3)))\n    model.add( BatchNormalization())\n    model.add( Conv2D(filters=64, kernel_size=(3,3), activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Conv2D(filters=64, kernel_size=5, strides=2, activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Dropout(0.2))\n    \n    model.add( Conv2D(filters=128, kernel_size=(3,3), activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Conv2D(filters=128, kernel_size=(3,3), activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Conv2D(filters=128, kernel_size=5, strides=2, activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Dropout(0.3))\n    \n    model.add( Conv2D(filters=256, kernel_size=(3,3), activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Conv2D(filters=256, kernel_size=(3,3), activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Conv2D(filters=256, kernel_size=5, strides=2, activation='relu', padding='same'))\n    model.add( BatchNormalization())\n    model.add( Dropout(0.4))\n    \n    model.add( Flatten())\n    model.add( Dense(512, activation='relu'))\n    \n    model.add( BatchNormalization())\n    model.add( Dropout(0.5))\n    model.add( Dense(10, activation='softmax'))\n    \n    return model\n\n#def make_model():\n    # Define the model\n #   model = Sequential()\n    \n #   model.add(Convolution2D(32,  kernel_size = 3, kernel_initializer='he_normal', activation='relu', input_shape = (32, 32, 3)))\n#    model.add(BatchNormalization())\n    \n #   model.add(Convolution2D(64, kernel_size = 3, kernel_initializer='he_normal', strides=1, activation='relu'))\n #   model.add(BatchNormalization())\n  #  model.add(MaxPooling2D((2, 2))) # 1st\n    \n  #  model.add(Convolution2D(64, kernel_size = 3, kernel_initializer='he_normal' , activation='relu'))\n  #  model.add(BatchNormalization())\n  #  model.add(MaxPooling2D((2, 2)))\n  #  model.add(Dropout(0.2))\n\n    \n  #  model.add(Convolution2D(128, kernel_size = 3, kernel_initializer='he_normal', strides=1 ,padding='same', activation='relu'))\n  #  model.add(BatchNormalization())\n   # model.add(MaxPooling2D((2, 2)))\n    \n   # model.add(Convolution2D(128, kernel_size = 3, kernel_initializer='he_normal' ,padding='same', activation='relu'))\n   # model.add(BatchNormalization())\n   # model.add(MaxPooling2D((2, 2)))\n    \n   # model.add(Convolution2D(64, kernel_size = 3,kernel_initializer='he_normal',padding='same', activation='relu'))\n   # model.add(BatchNormalization())\n   # model.add(MaxPooling2D((2, 2),padding='same'))\n   # model.add(Dropout(0.2))\n   # model.add(Flatten())\n    \n   # model.add(Dense(512,kernel_initializer='he_normal', activation = \"relu\"))\n    #model.add(Dropout(0.15))\n    \n   # model.add(Dense(256,kernel_initializer='he_normal', activation = \"relu\"))\n   # model.add(Dropout(0.1))\n    \n   # model.add(Dense(10, kernel_initializer='glorot_uniform',activation='softmax'))\n    \n#    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:39:00.693023Z","iopub.execute_input":"2022-09-20T12:39:00.693387Z","iopub.status.idle":"2022-09-20T12:39:00.891663Z","shell.execute_reply.started":"2022-09-20T12:39:00.693353Z","shell.execute_reply":"2022-09-20T12:39:00.890672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Scheduler of learning rate (decay with EPOCHS)","metadata":{}},{"cell_type":"code","source":"INIT_LR = 3e-3\nBATCH_SIZE = 64\nEPOCHS = 100\n\ndef lr_scheduler(epoch):\n    return INIT_LR * 0.9 ** epoch\n\n# callback for printing of actual learning rate used by optimizer\nclass LrHistory(keras.callbacks.Callback):\n    def on_epoch_begin(self, epoch, logs={}):\n        print(\"Learning rate:\", K.get_value(model.optimizer.lr))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:39:00.893248Z","iopub.execute_input":"2022-09-20T12:39:00.893913Z","iopub.status.idle":"2022-09-20T12:39:00.903287Z","shell.execute_reply.started":"2022-09-20T12:39:00.893861Z","shell.execute_reply":"2022-09-20T12:39:00.902383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configure the defined Model and sumarize the model archetecture","metadata":{}},{"cell_type":"code","source":"model = make_model()\nmodel.summary()\n#OPT = tf.keras.optimizers.Adamax(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name=\"Adamax\",)\n#model.compile(loss='categorical_crossentropy', optimizer= OPT, metrics=['accuracy'])  # report accuracy during training\nmodel.compile(loss='categorical_crossentropy', optimizer=tf.optimizers.Adamax(learning_rate=INIT_LR), metrics=['accuracy']) #,beta_1=0.9, beta_2=0.999, epsilon=1e-07, name=\"Adamax\"), metrics=['accuracy'])  # report accuracy during training","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:39:00.904898Z","iopub.execute_input":"2022-09-20T12:39:00.905332Z","iopub.status.idle":"2022-09-20T12:39:03.506022Z","shell.execute_reply.started":"2022-09-20T12:39:00.905297Z","shell.execute_reply":"2022-09-20T12:39:03.505083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata_generator = tf.keras.preprocessing.image.ImageDataGenerator(featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False,\n                                                                 zca_whitening=False,  rotation_range=20, width_shift_range=4,  height_shift_range=4,\n                                                                 brightness_range=[0.8,1.0], shear_range=0.0, zoom_range=0.1, channel_shift_range=0.0, \n                                                                 horizontal_flip=False,vertical_flip=False,rescale=1./255) #zoom_range=[0.2,1.0]\n\n\ntrain_generator = data_generator.flow(X_train, Y_train, BATCH_SIZE)\nsteps_per_epoch = np.ceil(X_train.shape[0] / BATCH_SIZE)\n\n#tqdm_callback = tfa.callbacks.TQDMProgressBar()\nmodel.fit(train_generator, steps_per_epoch=steps_per_epoch, epochs = 100) # , workers=8, use_multiprocessing=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T12:39:03.507873Z","iopub.execute_input":"2022-09-20T12:39:03.508488Z","iopub.status.idle":"2022-09-20T14:02:24.093286Z","shell.execute_reply.started":"2022-09-20T12:39:03.508451Z","shell.execute_reply":"2022-09-20T14:02:24.092182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Fitting model is, finding model parameters that in some sense explain the training data.","metadata":{}},{"cell_type":"code","source":"#tqdm_callback = tfa.callbacks.TQDMProgressBar()\n\n#model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=EPOCHS,\n#            callbacks=[keras.callbacks.LearningRateScheduler(lr_scheduler), \n#               LrHistory(), tqdm_callback], shuffle=True, verbose=0, initial_epoch=0)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:02:24.097229Z","iopub.execute_input":"2022-09-20T14:02:24.097527Z","iopub.status.idle":"2022-09-20T14:02:24.102272Z","shell.execute_reply.started":"2022-09-20T14:02:24.097498Z","shell.execute_reply":"2022-09-20T14:02:24.101064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving the trained model for further enhancement of the existing model in the future.","metadata":{}},{"cell_type":"code","source":"model.save('Project_Model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:02:24.103945Z","iopub.execute_input":"2022-09-20T14:02:24.104498Z","iopub.status.idle":"2022-09-20T14:02:24.328481Z","shell.execute_reply.started":"2022-09-20T14:02:24.104432Z","shell.execute_reply":"2022-09-20T14:02:24.327511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from keras.models import load_model\n#model= load_model('../output/kaggle/working/Project_Model.h5')\n#model= load_model('../input/project-model6/Project_Model.h5')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:02:24.330010Z","iopub.execute_input":"2022-09-20T14:02:24.330364Z","iopub.status.idle":"2022-09-20T14:02:24.334767Z","shell.execute_reply.started":"2022-09-20T14:02:24.330328Z","shell.execute_reply":"2022-09-20T14:02:24.333779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.register_unpack_format('7zip',['.7z'],unpack_7zarchive)\nshutil.unpack_archive('/kaggle/input/cifar-10/test.7z','/kaggle/temp/test')\nshutil.unregister_unpack_format('7zip')\ntest_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)\n\ntest_gen = test_datagen.flow_from_directory(directory='/kaggle/temp/test',target_size=(32,32),batch_size=64,class_mode=None,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:02:24.336392Z","iopub.execute_input":"2022-09-20T14:02:24.337010Z","iopub.status.idle":"2022-09-20T14:16:35.216290Z","shell.execute_reply.started":"2022-09-20T14:02:24.336974Z","shell.execute_reply":"2022-09-20T14:16:35.215271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)\n\n#test_gen = test_datagen.flow_from_directory(X_test,target_size=(32,32),batch_size=64,class_mode=None,shuffle=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:16:35.217829Z","iopub.execute_input":"2022-09-20T14:16:35.218193Z","iopub.status.idle":"2022-09-20T14:16:35.222289Z","shell.execute_reply.started":"2022-09-20T14:16:35.218155Z","shell.execute_reply":"2022-09-20T14:16:35.221253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making test predictions\n","metadata":{}},{"cell_type":"code","source":"test_gen.reset()\npredictions_vecs = model.predict(test_gen)\n\npredictions_final = np.argmax(predictions_vecs, axis=1)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:16:35.223917Z","iopub.execute_input":"2022-09-20T14:16:35.224559Z","iopub.status.idle":"2022-09-20T14:18:09.237618Z","shell.execute_reply.started":"2022-09-20T14:16:35.224511Z","shell.execute_reply":"2022-09-20T14:18:09.236625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cifar10_classes = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\nprint(cifar10_classes)\npredicted_classes=np.empty(shape=300000,dtype=np.dtype('U20'))\n\nind=0\nfor i in predictions_final.tolist():\n    predicted_classes[ind]=cifar10_classes[i]\n    ind=ind+1\n    \nfilenames_wo_ext = []\nfor fname in test_gen.filenames:\n    filenames_wo_ext.append(int(fname.split(sep=\"/\")[1].split(sep=\".\")[0])-1)\n\npredicted_classes_final = np.empty(shape=300000,dtype=np.dtype('U20'))\npredicted_classes_final[filenames_wo_ext]=predicted_classes","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:18:09.239101Z","iopub.execute_input":"2022-09-20T14:18:09.239487Z","iopub.status.idle":"2022-09-20T14:18:09.606106Z","shell.execute_reply.started":"2022-09-20T14:18:09.239451Z","shell.execute_reply":"2022-09-20T14:18:09.605087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating a Kaggle csv Submission file.","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('../input/cifar-10/sampleSubmission.csv',header='infer')\nsub.info()\n\nsub['label'] = predicted_classes_final\nsub.to_csv('Submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T14:18:09.607698Z","iopub.execute_input":"2022-09-20T14:18:09.608112Z","iopub.status.idle":"2022-09-20T14:18:10.003041Z","shell.execute_reply.started":"2022-09-20T14:18:09.608074Z","shell.execute_reply":"2022-09-20T14:18:10.001535Z"},"trusted":true},"execution_count":null,"outputs":[]}]}