{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30158,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"CIFAR 10 Image Classification using transfer learning. 10 classes. 50,000 images in training, 5000 images of each class. 10,000 images in the test set. To avoid leaks, test set here has 3 lacs images (290000 junk images, however they are ignored in scoring). Also, test images are tweaked to avoid look up by file hash\n\nThis Notebook:\n- Transfer learning\n- Generator and flow from directory is used, which are very important when we work with large size of data. It also supports data augmentation. \n\nPlease Upvote, if you find it useful.","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd # for reading csv file","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-16T07:01:12.216938Z","iopub.execute_input":"2022-03-16T07:01:12.217652Z","iopub.status.idle":"2022-03-16T07:01:12.245431Z","shell.execute_reply.started":"2022-03-16T07:01:12.217559Z","shell.execute_reply":"2022-03-16T07:01:12.244849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train and test images are .7z archives, we need to unpack them\n!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:01:47.114363Z","iopub.execute_input":"2022-03-16T07:01:47.114922Z","iopub.status.idle":"2022-03-16T07:02:00.004086Z","shell.execute_reply.started":"2022-03-16T07:01:47.114885Z","shell.execute_reply":"2022-03-16T07:02:00.003313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from py7zr import unpack_7zarchive\nimport shutil\n\n#Before using, we need to register unpack format\nshutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:02:04.053569Z","iopub.execute_input":"2022-03-16T07:02:04.054111Z","iopub.status.idle":"2022-03-16T07:02:04.201602Z","shell.execute_reply.started":"2022-03-16T07:02:04.054072Z","shell.execute_reply":"2022-03-16T07:02:04.200899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#unpack train images in /kaggle/working or /kaggle/temp\nshutil.unpack_archive('/kaggle/input/cifar-10/train.7z', '/kaggle/temp/')","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:02:06.532927Z","iopub.execute_input":"2022-03-16T07:02:06.533207Z","iopub.status.idle":"2022-03-16T07:02:48.857168Z","shell.execute_reply.started":"2022-03-16T07:02:06.533176Z","shell.execute_reply":"2022-03-16T07:02:48.856457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#just checking the filenames\nimport os\ni=0\nfor root, dirnames,fnames in os.walk(\"/kaggle/temp/train\"):\n    for fname in sorted(fnames):\n        print(fname)\n        print(os.path.join(root,fname))\n        i=i+1\n        if(i==50):\n            break\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:03:34.49609Z","iopub.execute_input":"2022-03-16T07:03:34.496344Z","iopub.status.idle":"2022-03-16T07:03:34.590888Z","shell.execute_reply.started":"2022-03-16T07:03:34.496316Z","shell.execute_reply":"2022-03-16T07:03:34.590207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we are going to use flow_from_directory, it requires a separate sub folder for each class under train folder.\n#images belonging to a class need to be within the sub folder corresponding to them\n#therefore, first fetching the class names from trainLabels.csv\n\ntrain_labels=pd.read_csv(\"/kaggle/input/cifar-10/trainLabels.csv\", header='infer')\n\n#unique labels\nclasses=train_labels['label'].unique()\n\n#confirming\nprint(classes)\n\n#os module will be required to move train images in the correct sub folder\nimport os\n\nif not os.path.exists(\"/kaggle/temp/validation/\"):\n    os.mkdir(\"/kaggle/temp/validation/\")\n\nparent_path_train=\"/kaggle/temp/train/\"\nparent_path_validation=\"/kaggle/temp/validation/\"\nparent_path_test=\"/kaggle/temp/test/\"\n\nfor class1 in classes:\n    path_train=os.path.join(parent_path_train,class1)\n    if not os.path.exists(path_train):\n        os.mkdir(path_train)\n    path_validation=os.path.join(parent_path_validation,class1)\n    if not os.path.exists(path_validation):\n        os.mkdir(path_validation)\n\n#Now we are going to move images in the correct sub-directories\n#At the same time, we are going to divide these images in train and validation set\nfor (int_ind,row) in train_labels.iterrows():\n    id=str(row[\"id\"])+\".png\" #all images are .png images\n    source_path=os.path.join(parent_path_train,id)\n    \n    p=np.random.random() #generating a random number between [0,1)\n    if p <= 0.8: #for approximate 80/20 split\n        target_path=os.path.join(parent_path_train,row[\"label\"],id)\n        os.replace(source_path, target_path) #note replace, this means we are moving and not copying files\n    else:\n        target_path=os.path.join(parent_path_validation,row[\"label\"],id)\n        os.replace(source_path, target_path)\n\n#confirming sub-directories\n!ls /kaggle/temp/train\n!ls /kaggle/temp/validation\n\n#you may confirm files\n#!ls /kaggle/working/validation/airplane","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:03:49.539762Z","iopub.execute_input":"2022-03-16T07:03:49.540214Z","iopub.status.idle":"2022-03-16T07:03:54.303494Z","shell.execute_reply.started":"2022-03-16T07:03:49.540178Z","shell.execute_reply":"2022-03-16T07:03:54.302765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Now import for models and generator\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom keras.applications.xception import Xception, preprocess_input\nfrom keras import Sequential\nfrom keras.layers import Dense, Activation, BatchNormalization, GlobalAveragePooling2D,Dropout\nfrom keras.utils.np_utils import to_categorical\nfrom keras.callbacks import ReduceLROnPlateau\n\nfrom keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2022-03-16T07:06:00.36443Z","iopub.execute_input":"2022-03-16T07:06:00.365172Z","iopub.status.idle":"2022-03-16T07:06:00.371616Z","shell.execute_reply.started":"2022-03-16T07:06:00.365133Z","shell.execute_reply":"2022-03-16T07:06:00.370839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create instance of xception wihtout top\nxception_wo_top=Xception(include_top=False, input_shape=(291, 291, 3), weights='imagenet')\nxception_wo_top.summary()\n\nfor layer in xception_wo_top.layers:\n    layer.trainable=True #by default, True but you can make it false by specifying False","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:28:36.220836Z","iopub.execute_input":"2022-03-16T10:28:36.221753Z","iopub.status.idle":"2022-03-16T10:28:37.343268Z","shell.execute_reply.started":"2022-03-16T10:28:36.2217Z","shell.execute_reply":"2022-03-16T10:28:37.34256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = Sequential()\nclassifier.add(xception_wo_top)\nclassifier.add(GlobalAveragePooling2D())\nclassifier.add(Dropout(0.3))\nclassifier.add(Dense(10, activation='softmax'))\n\nclassifier.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nclassifier.summary()\n\nfor l in classifier.layers:\n    print(l.trainable)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:29:36.669338Z","iopub.execute_input":"2022-03-16T10:29:36.67001Z","iopub.status.idle":"2022-03-16T10:29:36.993781Z","shell.execute_reply.started":"2022-03-16T10:29:36.66997Z","shell.execute_reply":"2022-03-16T10:29:36.992982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Set how we plan to reduce learning rate on plateau\nreduceLROnPlateau = ReduceLROnPlateau(monitor='val_acc', \n                                patience=3,\n                                verbose=1, \n                                factor=0.5,\n                                min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:29:45.119355Z","iopub.execute_input":"2022-03-16T10:29:45.120063Z","iopub.status.idle":"2022-03-16T10:29:45.125282Z","shell.execute_reply.started":"2022-03-16T10:29:45.120027Z","shell.execute_reply":"2022-03-16T10:29:45.124382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating instance and specifying augmentation\ntrain_datagen=ImageDataGenerator(featurewise_center=False,\n                         samplewise_center=False,\n                         featurewise_std_normalization=False,\n                         samplewise_std_normalization=False,\n                         zca_whitening=False,\n                         rotation_range=10,\n                         zoom_range=0.1,\n                         width_shift_range=0.1,\n                         height_shift_range=0.1,\n                         horizontal_flip=False,\n                         vertical_flip=False,\n                         preprocessing_function=preprocess_input,\n                         rescale=None\n                        )\ntrain_gen=train_datagen.flow_from_directory(directory='/kaggle/temp/train/', target_size=(192,192), batch_size=64) \n\nvalidation_datagen=ImageDataGenerator(preprocessing_function=preprocess_input)\n\nvalid_gen=validation_datagen.flow_from_directory(directory='/kaggle/temp/validation/', target_size=(192,192), batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:37:07.013704Z","iopub.execute_input":"2022-03-16T10:37:07.015308Z","iopub.status.idle":"2022-03-16T10:37:08.361416Z","shell.execute_reply.started":"2022-03-16T10:37:07.015231Z","shell.execute_reply":"2022-03-16T10:37:08.360612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier.fit(train_gen, epochs=10, callbacks=[reduceLROnPlateau], validation_data=valid_gen, steps_per_epoch = train_gen.n//train_gen.batch_size,\n                    validation_steps = valid_gen.n//valid_gen.batch_size, use_multiprocessing=True, workers=8)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:37:12.238106Z","iopub.execute_input":"2022-03-16T10:37:12.23866Z","iopub.status.idle":"2022-03-16T10:45:01.334463Z","shell.execute_reply.started":"2022-03-16T10:37:12.23862Z","shell.execute_reply":"2022-03-16T10:45:01.327825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#unpacking test images, there are 3 lacs images. This will take some time\nshutil.unpack_archive('/kaggle/input/cifar-10/test.7z', '/kaggle/temp/test')","metadata":{"execution":{"iopub.status.busy":"2022-03-16T08:46:22.218352Z","iopub.execute_input":"2022-03-16T08:46:22.218699Z","iopub.status.idle":"2022-03-16T09:00:30.501302Z","shell.execute_reply.started":"2022-03-16T08:46:22.218661Z","shell.execute_reply":"2022-03-16T09:00:30.50028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#unregister unpack format, we are done with it\nshutil.unregister_unpack_format('7zip')#, ['.7z'], unpack_7zarchive)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T09:08:24.040614Z","iopub.execute_input":"2022-03-16T09:08:24.041157Z","iopub.status.idle":"2022-03-16T09:08:24.045357Z","shell.execute_reply.started":"2022-03-16T09:08:24.041118Z","shell.execute_reply":"2022-03-16T09:08:24.044467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test data generator, we are only normalizing test images\ntest_datagen=ImageDataGenerator(preprocessing_function=preprocess_input)\n\n#we don't have labels for test images and therefore we need to set class_mode=None\n#because of this, generator will only return batches of images\ntest_gen=test_datagen.flow_from_directory(directory='/kaggle/temp/test', target_size=(291,291), batch_size=64, class_mode=None, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:01:26.618583Z","iopub.execute_input":"2022-03-16T10:01:26.619038Z","iopub.status.idle":"2022-03-16T10:01:33.088011Z","shell.execute_reply.started":"2022-03-16T10:01:26.619005Z","shell.execute_reply":"2022-03-16T10:01:33.087299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_gen.reset()\nprediction_vectors=classifier.predict(test_gen)\n\n#One way of esembling, average predictions for 3 models and then use argmax to decide the\n#prediction with max probability\n#average_prediction_vectors=(prediction_vectors3+prediction_vectors5+prediction_vectors7)/3.\n#predictions_final=np.argmax(average_prediction_vectors, axis=1)\n\npredictions_final=np.argmax(prediction_vectors, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:01:37.248623Z","iopub.execute_input":"2022-03-16T10:01:37.249073Z","iopub.status.idle":"2022-03-16T10:05:22.036684Z","shell.execute_reply.started":"2022-03-16T10:01:37.249034Z","shell.execute_reply":"2022-03-16T10:05:22.035936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we will require to convert numeric predictions to text labels\n\n#first let us list dictionary of mapping class text label and numeric mapping\nprint(type(train_gen.class_indices)) #dictionary where key is text label and value is numeric label\nprint(train_gen.class_indices) \n\n#make numeric label the key and text label value\nclasses={value:key for (key, value) in train_gen.class_indices.items()}\nprint(classes)\n\n\n#now let us convert predicted numeric label to text label\n\n#first set up the numpy array. U for unicode, 20 max size\npredicted_classes=np.empty(shape=300000, dtype=np.dtype('U20'))\n\nind=0 #index into above array\nfor i in predictions_final.tolist():\n    predicted_classes[ind]=classes[i]\n    ind=ind+1\n\n#test generator has read images in alphanumeric order\n#and our submission file needs predictions in numeric order\n\nfilenames_wo_ext=[] #for rearranging array\nfor fname in test_gen.filenames:\n    filenames_wo_ext.append(int(fname.split(sep=\"/\")[1].split(sep=\".\")[0])-1)\n\npredicted_classes_final=np.empty(shape=300000, dtype=np.dtype('U20'))\npredicted_classes_final[filenames_wo_ext]=predicted_classes #rearranging","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:05:33.54125Z","iopub.execute_input":"2022-03-16T10:05:33.541516Z","iopub.status.idle":"2022-03-16T10:05:33.999381Z","shell.execute_reply.started":"2022-03-16T10:05:33.541485Z","shell.execute_reply":"2022-03-16T10:05:33.998493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading sample submission file\nsub=pd.read_csv('/kaggle/input/cifar-10/sampleSubmission.csv', header='infer')\n\n#just ensuring the data type\nsub.info()\n\n#overwriting the label column with prediction\nsub['label']=predicted_classes_final\n\n#again confirming the data type\nsub.info()\n\n#writing back as submission.csv\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-16T10:05:41.911321Z","iopub.execute_input":"2022-03-16T10:05:41.912053Z","iopub.status.idle":"2022-03-16T10:05:42.578048Z","shell.execute_reply.started":"2022-03-16T10:05:41.912014Z","shell.execute_reply":"2022-03-16T10:05:42.577191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That is it.\n\nPlease Upvote, if you find it useful.","metadata":{}}]}