{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nfrom tensorflow import keras\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense, Conv2D, Flatten, BatchNormalization, Activation\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n#from scipy.stats import mode\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"3814fa8f-1c84-401f-8c8f-1dbb48f6e274","_cell_guid":"14185176-0805-4eb6-ae91-1b169e9864be","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:04.097446Z","iopub.execute_input":"2022-03-26T05:07:04.098338Z","iopub.status.idle":"2022-03-26T05:07:10.990267Z","shell.execute_reply.started":"2022-03-26T05:07:04.098227Z","shell.execute_reply":"2022-03-26T05:07:10.989278Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install np_utils","metadata":{"_uuid":"9e53ce84-c56a-441d-a995-37c5c5aebb25","_cell_guid":"37bdffb1-7f32-4510-b698-66c7a997268c","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:10.994642Z","iopub.execute_input":"2022-03-26T05:07:10.994872Z","iopub.status.idle":"2022-03-26T05:07:11.002817Z","shell.execute_reply.started":"2022-03-26T05:07:10.994841Z","shell.execute_reply":"2022-03-26T05:07:11.0015Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install -q efficientnet\n# !pip install tensorflow-addons==0.9.1","metadata":{"_uuid":"3febe761-15d8-427d-9eca-d5c27f9b770e","_cell_guid":"3642794a-642f-4d58-9267-6fe868dae774","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:11.005297Z","iopub.execute_input":"2022-03-26T05:07:11.005677Z","iopub.status.idle":"2022-03-26T05:07:11.022583Z","shell.execute_reply.started":"2022-03-26T05:07:11.005632Z","shell.execute_reply":"2022-03-26T05:07:11.021544Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_model3():\n    model=Sequential()\n    model.add( Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=48, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=80, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=96, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=112, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=144, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=160, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=176, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add(Flatten())\n    \n    model.add(Dense(units=10))\n    model.add(BatchNormalization())\n    model.add(Activation('softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","metadata":{"_uuid":"6bb16f69-fdf7-40b4-a82b-edd362a61e42","_cell_guid":"e6affe76-bb46-45e4-82c4-e71b8a1c345e","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:11.026821Z","iopub.execute_input":"2022-03-26T05:07:11.027068Z","iopub.status.idle":"2022-03-26T05:07:11.052054Z","shell.execute_reply.started":"2022-03-26T05:07:11.027037Z","shell.execute_reply":"2022-03-26T05:07:11.05111Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_model5():\n    model=Sequential()\n    \n    model.add( Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=64, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=96, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False ) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=128, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=160, kernel_size=(5,5), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n        \n    model.add(Flatten())\n    \n    model.add(Dense(units=10))\n    model.add(BatchNormalization())\n    model.add(Activation('softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","metadata":{"_uuid":"b0609870-06d3-405d-a0d1-db4ee8388325","_cell_guid":"dd243cac-167c-4ea2-8c52-22a504a8595d","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:11.054687Z","iopub.execute_input":"2022-03-26T05:07:11.055366Z","iopub.status.idle":"2022-03-26T05:07:11.068931Z","shell.execute_reply.started":"2022-03-26T05:07:11.05532Z","shell.execute_reply":"2022-03-26T05:07:11.067935Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_model7():\n    model=Sequential()\n    \n    model.add( Conv2D(filters=48, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(X_train.shape[1:])) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=96, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=144, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    \n    model.add( Conv2D(filters=192, kernel_size=(7,7), strides=(1,1), padding='valid', activation=None, use_bias=False) )\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n        \n    model.add(Flatten())\n    \n    model.add(Dense(units=10))\n    model.add(BatchNormalization())\n    model.add(Activation('softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","metadata":{"_uuid":"a581736f-e03d-4631-bf30-daab9b206ec9","_cell_guid":"4f8e697d-0d98-4757-b82e-38d53e66d070","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:11.072575Z","iopub.execute_input":"2022-03-26T05:07:11.07281Z","iopub.status.idle":"2022-03-26T05:07:11.086608Z","shell.execute_reply.started":"2022-03-26T05:07:11.072778Z","shell.execute_reply":"2022-03-26T05:07:11.085614Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_full=pd.read_csv('/kaggle/input/digit-recognizer/train.csv', header='infer').values\nX_test=pd.read_csv('/kaggle/input/digit-recognizer/test.csv', header='infer').values\n\n#Separate label and images from the training data\nX_train=X_train_full[:,1:]\ny_train=X_train_full[:,0]\n\n#Normalize train and test images\nX_train = (X_train.astype(np.float32) - 127.5)/127.5\nX_test = (X_test.astype(np.float32) - 127.5)/127.5","metadata":{"_uuid":"f5958d51-856c-464f-b97d-5f75528bfcbd","_cell_guid":"a89d263a-90d3-4553-8f41-d98d6c88819b","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:11.0883Z","iopub.execute_input":"2022-03-26T05:07:11.088753Z","iopub.status.idle":"2022-03-26T05:07:17.603235Z","shell.execute_reply.started":"2022-03-26T05:07:11.088708Z","shell.execute_reply":"2022-03-26T05:07:17.602244Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train=X_train.reshape(-1,28,28,1)\nX_test=X_test.reshape(-1,28,28,1)\n\n#One-hot encode class labels\ny_train_vectors=to_categorical(y_train)\n\nprint(X_train.shape)\nprint(X_test.shape)","metadata":{"_uuid":"a6d9ec88-1a00-4e5f-832c-b4a6b3728580","_cell_guid":"e4450f9f-84e8-493f-b7e1-bdcaf8959e8e","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:17.606366Z","iopub.execute_input":"2022-03-26T05:07:17.606925Z","iopub.status.idle":"2022-03-26T05:07:17.618053Z","shell.execute_reply.started":"2022-03-26T05:07:17.606874Z","shell.execute_reply":"2022-03-26T05:07:17.616764Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val= train_test_split(X_train, y_train_vectors, test_size=0.2, random_state=2)\n\n#Create instance of 3 CNNs\nmodel3=my_model3()\nmodel5=my_model5()\nmodel7=my_model7()","metadata":{"_uuid":"29143227-ceae-49cd-bf18-c3d932462ebb","_cell_guid":"94be00dd-7d16-4948-b493-6b763714c437","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:17.61991Z","iopub.execute_input":"2022-03-26T05:07:17.620333Z","iopub.status.idle":"2022-03-26T05:07:21.529621Z","shell.execute_reply.started":"2022-03-26T05:07:17.620286Z","shell.execute_reply":"2022-03-26T05:07:21.528691Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(featurewise_center=False,\n                             samplewise_center=False,\n                             featurewise_std_normalization=False,\n                             samplewise_std_normalization=False,\n                             zca_whitening=False,\n                             rotation_range=10,\n                             zoom_range=0.1,\n                             width_shift_range=0.1,\n                             height_shift_range=0.1,\n                             horizontal_flip=False,\n                             vertical_flip=False\n                            )\n\n#Use flow method to pass images to fit method in the batches of size 120\ntrain_generator = train_datagen.flow(X_train, y_train,\n                                     batch_size=120,\n                                     shuffle=True)\n\nval_datagen = ImageDataGenerator()\nval_generator = val_datagen.flow(X_val, y_val,\n                                 batch_size=120,\n                                 shuffle=True)","metadata":{"_uuid":"f4bc2f5c-20d6-4e5d-930c-9f57a9bc4d78","_cell_guid":"848afaeb-baff-4d89-aca6-5f0f8a81c0dc","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:21.534319Z","iopub.execute_input":"2022-03-26T05:07:21.53508Z","iopub.status.idle":"2022-03-26T05:07:21.545518Z","shell.execute_reply.started":"2022-03-26T05:07:21.535044Z","shell.execute_reply":"2022-03-26T05:07:21.544608Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduceLROnPlateau = ReduceLROnPlateau(monitor='val_acc', \n                                patience=3,\n                                verbose=1, \n                                factor=0.5,\n                                min_lr=0.00001)","metadata":{"_uuid":"e35f1d80-8270-4b2b-b66c-b1aedf58f9d8","_cell_guid":"150f0584-e71f-4f18-9f52-c9a891ca6ab0","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:21.549027Z","iopub.execute_input":"2022-03-26T05:07:21.549339Z","iopub.status.idle":"2022-03-26T05:07:21.56328Z","shell.execute_reply.started":"2022-03-26T05:07:21.549305Z","shell.execute_reply":"2022-03-26T05:07:21.562037Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras import datasets, layers, models, regularizers\n\n# # Hyperparameters\n# epochs = 15\n# drop = 0.1\n\n# # Create a new model\n# def CNN():\n#     reg = regularizers.l2(1e-3)\n\n#     # Convolutional layers\n#     model = models.Sequential()\n#     model.add(layers.Conv2D(512, (5, 5), activation='relu', kernel_regularizer=reg, bias_regularizer=reg, input_shape=shape))\n#     model.add(layers.MaxPooling2D((2, 2)))\n#     model.add(layers.Dropout(drop))\n#     model.add(layers.Conv2D(512, (3, 3), activation='relu', kernel_regularizer=reg, bias_regularizer=reg))\n#     model.add(layers.MaxPooling2D((2, 2)))\n#     model.add(layers.Dropout(drop))\n#     model.add(layers.Conv2D(512, (3, 3), activation='relu', kernel_regularizer=reg, bias_regularizer=reg))\n\n#     # Dense layers\n#     model.add(layers.Flatten())\n#     model.add(layers.Dropout(drop))\n#     model.add(layers.Dense(512, activation='relu'))\n#     model.add(layers.Dense(10))\n\n#     model.compile(optimizer='adam',\n#                   loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n#                   metrics=['accuracy'])\n#     return model\n\n# CNN().summary()","metadata":{"_uuid":"7156d357-4c8e-41d8-b4bb-3a16c2bdd288","_cell_guid":"7b33dd81-04a3-41b4-a46d-5eabf0c7005c","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:21.566801Z","iopub.execute_input":"2022-03-26T05:07:21.567077Z","iopub.status.idle":"2022-03-26T05:07:21.574857Z","shell.execute_reply.started":"2022-03-26T05:07:21.567045Z","shell.execute_reply":"2022-03-26T05:07:21.573836Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# case = CNN()\n# history = case.fit(image(train), y_train, epochs=epochs)","metadata":{"_uuid":"1d5fe3de-15b4-4e6d-a037-699a61a6f2a9","_cell_guid":"b68408b6-4e95-4bb2-82b4-6965d0d1bc97","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:21.577904Z","iopub.execute_input":"2022-03-26T05:07:21.578149Z","iopub.status.idle":"2022-03-26T05:07:21.590087Z","shell.execute_reply.started":"2022-03-26T05:07:21.57812Z","shell.execute_reply":"2022-03-26T05:07:21.589086Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3.fit(train_generator, epochs=100, callbacks=[reduceLROnPlateau], validation_data=val_generator)\nmodel5.fit(train_generator, epochs=100, callbacks=[reduceLROnPlateau], validation_data=val_generator)\nmodel7.fit(train_generator, epochs=100, callbacks=[reduceLROnPlateau], validation_data=val_generator)\n\nprediction_vectors3=model3.predict(X_test)\nprediction_vectors5=model5.predict(X_test)\nprediction_vectors7=model7.predict(X_test)\n\nprint(prediction_vectors3.shape)\nprint(prediction_vectors5.shape)\nprint(prediction_vectors7.shape)","metadata":{"_uuid":"11f6516a-b12c-428c-ab06-557544fa92e4","_cell_guid":"ba3d372e-4053-4807-b7a8-929e9033b2cb","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T05:07:21.591912Z","iopub.execute_input":"2022-03-26T05:07:21.592227Z","iopub.status.idle":"2022-03-26T06:23:36.652017Z","shell.execute_reply.started":"2022-03-26T05:07:21.592185Z","shell.execute_reply":"2022-03-26T06:23:36.650949Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#One way of esembling, average predictions for 3 models and then use argmax to decide the\n#prediction with max probability\naverage_prediction_vectors=(prediction_vectors3+prediction_vectors5+prediction_vectors7)/3.\npredictions_final=np.argmax(average_prediction_vectors, axis=1)","metadata":{"_uuid":"47f3b331-b83d-40b4-a2dc-76e95367c2e4","_cell_guid":"f9e60760-1f64-49cf-a3b0-e983228dc183","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T06:27:49.992703Z","iopub.execute_input":"2022-03-26T06:27:49.993009Z","iopub.status.idle":"2022-03-26T06:27:50.001263Z","shell.execute_reply.started":"2022-03-26T06:27:49.992975Z","shell.execute_reply":"2022-03-26T06:27:50.000119Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Read sample_submission.csv in dataframe sub\nsub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\n\n#Overwrite labels in dataframe sub\nsub[\"Label\"] = predictions_final\n\n#Write updated dataframes as submission.csv\nsub.to_csv('19BCE226_Prac2.csv', index=False)","metadata":{"_uuid":"6c7eb7eb-4055-4c0a-bc40-c3af07da62a1","_cell_guid":"b1e3c202-07f3-4ae6-899b-a1053dd491ef","collapsed":false,"execution":{"iopub.status.busy":"2022-03-26T06:27:54.138949Z","iopub.execute_input":"2022-03-26T06:27:54.139702Z","iopub.status.idle":"2022-03-26T06:27:54.202922Z","shell.execute_reply.started":"2022-03-26T06:27:54.139658Z","shell.execute_reply":"2022-03-26T06:27:54.202029Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}