{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nfrom tensorflow import keras\nfrom keras import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten, BatchNormalization, Activation\nfrom keras.utils.np_utils import to_categorical\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\n#from scipy.stats import mode\nfrom sklearn.model_selection import train_test_split\n\n#Define model with 3 x 3 valid convolution, kernel_size=3, stride 1, and ReLU activation. \n#Also use BatchNormalization\ndef my_model3():\n    model=Sequential()\n    model.add( Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=48, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=80, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=96, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=112, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=144, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=160, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=176, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add(Flatten())\n    \n    model.add(Dense(units=10))\n    model.add(BatchNormalization())\n    model.add(Activation('softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model\n\n#Define model with 5 x 5 valid convolution, kernel_size=3, stride 1, and ReLU activation. \n#Also use BatchNormalization\ndef my_model5():\n    model=Sequential()\n    \n    model.add( Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=64, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=96, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False ) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=128, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=160, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n        \n    model.add(Flatten())\n    \n    model.add(Dense(units=10))\n    model.add(BatchNormalization())\n    model.add(Activation('softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model\n\n#Define model with 7 x 7 valid convolution, kernel_size=3, stride 1, and ReLU activation. \n#Also use BatchNormalization\ndef my_model7():\n    model=Sequential()\n    \n    model.add( Conv2D(filters=48, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=96, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=144, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=192, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n        \n    model.add(Flatten())\n    \n    model.add(Dense(units=10))\n    model.add(BatchNormalization())\n    model.add(Activation('softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model\n\n'''\n#The following function is useful if you want to implement ensemble throug majority vote\ndef mostCommon(mostC):\n    val, count = mode(mostC, axis=1)\n    return val.ravel()#.tolist() '''\n\n#Read training and test data\nX_train_full=pd.read_csv('/kaggle/input/digit-recognizer/train.csv', header='infer').values\nX_test=pd.read_csv('/kaggle/input/digit-recognizer/test.csv', header='infer').values\n\n#Separate label and images from the training data\nX_train=X_train_full[:,1:]\ny_train=X_train_full[:,0]\n\n#Normalize train and test images\nX_train = (X_train.astype(np.float32) - 127.5)/127.5\nX_test = (X_test.astype(np.float32) - 127.5)/127.5\n\n#If you wish to normalize intensities in the range of 0 to 1 use following\n#X_train=X_train/255.\n#X_test=X_test/255.\n\n#delete X_train_full, you don't need it further\ndel X_train_full\n\n#Reshpae train and test images from 784 to 28 x 28 x 1\nX_train=X_train.reshape(-1,28,28,1)\nX_test=X_test.reshape(-1,28,28,1)\n\n#One-hot encode class labels\ny_train_vectors=to_categorical(y_train)\n\nprint(X_train.shape)\nprint(X_test.shape)\n\nX_train, X_val, y_train, y_val= train_test_split(X_train, y_train_vectors, test_size=0.2, random_state=2)\n\n#Create instance of 3 CNNs\nmodel3=my_model3()\nmodel5=my_model5()\nmodel7=my_model7()\n\n\n#Create instance of ImageDataGenerator for augmenting training images.\n#Augmentation can help avoid overfitting\n#We are using rotation_range=10,zoom_range=0.1, width_shift_range=0.1, height_shift_range=0.1. \n#Nothing else for augmentation\n\ntrain_datagen = ImageDataGenerator(featurewise_center=False,\n                             samplewise_center=False,\n                             featurewise_std_normalization=False,\n                             samplewise_std_normalization=False,\n                             zca_whitening=False,\n                             rotation_range=10,\n                             zoom_range=0.1,\n                             width_shift_range=0.1,\n                             height_shift_range=0.1,\n                             horizontal_flip=False,\n                             vertical_flip=False\n                            )\n\n#Use flow method to pass images to fit method in the batches of size 120\ntrain_generator = train_datagen.flow(X_train, y_train,\n                                     batch_size=120,\n                                     shuffle=True)\n\nval_datagen = ImageDataGenerator()\nval_generator = val_datagen.flow(X_val, y_val,\n                                 batch_size=120,\n                                 shuffle=True)\n\n#Set how we plan to reduce learning rate on plateau\nreduceLROnPlateau = ReduceLROnPlateau(monitor='val_acc', \n                                patience=3,\n                                verbose=1, \n                                factor=0.5,\n                                min_lr=0.00001)\n\n\n#fit 3 CNNs\nmodel3.fit(train_generator, epochs=150, callbacks=[reduceLROnPlateau], validation_data=val_generator)\nmodel5.fit(train_generator, epochs=150, callbacks=[reduceLROnPlateau], validation_data=val_generator)\nmodel7.fit(train_generator, epochs=150, callbacks=[reduceLROnPlateau], validation_data=val_generator)\n\n#Use 3-trained CNNs to make predictions. \n#Each prediction varialbe is a matrix of size 28K x 10 as there are 10 classes\nprediction_vectors3=model3.predict(X_test)\nprediction_vectors5=model5.predict(X_test)\nprediction_vectors7=model7.predict(X_test)\n\nprint(prediction_vectors3.shape)\nprint(prediction_vectors5.shape)\nprint(prediction_vectors7.shape)\n\n#One way of esembling, average predictions for 3 models and then use argmax to decide the\n#prediction with max probability\naverage_prediction_vectors=(prediction_vectors3+prediction_vectors5+prediction_vectors7)/3.\npredictions_final=np.argmax(average_prediction_vectors, axis=1)\n\n'''\nAnother way of ensembling\nDecide prediction from individual model and then take the majority vote\n\n#Following 3 lines decide prediction from individual models, each prediction variable will be \n#now vector of size 28K\npredictions3=np.argmax(prediction_vectors3,axis=1)\npredictions5=np.argmax(prediction_vectors5,axis=1)\npredictions7=np.argmax(prediction_vectors7,axis=1)\n\nprint(predictions3.shape)\nprint(predictions5.shape)\nprint(predictions7.shape)\n\n#Combine predictions from individual model in 1 matrix, \n#number of rows will be 28K but now number of columns is 3\npredictions=np.stack([predictions3,predictions5,predictions7], axis=1)\n\n#mostCommon is the function written to take the majority vote\n#After call to it, predictions_final is back to vector of size 28K\npredictions_final=mostCommon(predictions)\nprint(predictions_final.shape)\n\n'''\n\n#Read sample_submission.csv in dataframe sub\nsub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\n\n#Overwrite labels in dataframe sub\nsub[\"Label\"] = predictions_final\n\n#Write updated dataframes as submission.csv\nsub.to_csv('submission.csv',index=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-11T03:16:30.407562Z","iopub.execute_input":"2022-03-11T03:16:30.408385Z"},"trusted":true},"execution_count":null,"outputs":[]}]}