{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory+\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-12T20:12:55.742805Z","iopub.execute_input":"2021-06-12T20:12:55.743178Z","iopub.status.idle":"2021-06-12T20:12:55.753018Z","shell.execute_reply.started":"2021-06-12T20:12:55.743134Z","shell.execute_reply":"2021-06-12T20:12:55.751881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a very beginner friendly kernel and my first kernel . I hope it helps you as much as i learned while implementing it , So lets Begin.","metadata":{}},{"cell_type":"markdown","source":"Notebooks that helped me , please give them a read .\nhttps://www.kaggle.com/poonaml/deep-neural-network-keras-way#Import-all-required-libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom keras.utils import to_categorical\nfrom keras.layers import *\nfrom sklearn.model_selection import StratifiedShuffleSplit \nimport numpy as np\nfrom keras.preprocessing import image","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-06-12T20:12:56.277119Z","iopub.execute_input":"2021-06-12T20:12:56.277463Z","iopub.status.idle":"2021-06-12T20:13:01.784649Z","shell.execute_reply.started":"2021-06-12T20:12:56.277434Z","shell.execute_reply":"2021-06-12T20:13:01.783877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We begin by importing libraries that i would use generally. Also i would recommend all beginners to keep all their imports at one place for ease and presentation wise too .","metadata":{}},{"cell_type":"code","source":"def load_data():\n    path = '/kaggle/input/digit-recognizer/'\n    train = pd.read_csv(path+'train.csv')\n    test = pd.read_csv(path+'test.csv')\n    train_X,train_y = train.drop(columns=['label']),train['label']\n    train_X = np.array(train_X)\n    \n    train_y = np.array(train_y)\n    \n    test_X = test\n    test_X = np.array(test_X)\n    print(train_X.shape)\n    train_X = train_X.reshape((train_X.shape[0],28,28,1))\n    print(train_X.shape)\n    test_X = test_X.reshape((test_X.shape[0],28,28,1))\n    train_y = to_categorical(train_y)\n    return train_X, train_y,test_X\ndef normalize(trainX,testX):\n    trainX_trans = trainX / 255\n    testX_trans = testX / 255\n    return trainX_trans,testX_trans\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Flatten())\n    model.add(Dense(64, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\n\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:01.786369Z","iopub.execute_input":"2021-06-12T20:13:01.786712Z","iopub.status.idle":"2021-06-12T20:13:01.801374Z","shell.execute_reply.started":"2021-06-12T20:13:01.786681Z","shell.execute_reply":"2021-06-12T20:13:01.800511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I have made functions that will help me load my data , normalise it , use encoding . So that i dont have to do same thing over and over again . This practise helped me to monitor more things at once and keep my code clean.","metadata":{}},{"cell_type":"markdown","source":"The load data function simply loads the dataset. \n\nI also reshaped the dataset's (which was converted into numpy array as they are fast and easy to deal) each sameple from 784 into 28 x 28 x 1 shape. For those who dont know the reason for this step can look at this article which exlain in detail and other details about convulotion layer.\nhttps://towardsdatascience.com/applied-deep-learning-part-4-convolutional-neural-networks-584bc134c1e2\n\nI used to_categorical to implement one hot encoding in my labels .(It made 3 to a array (0,0,0,1,0,0,0,0,0,0) ).\n\nAnd Normalize function normalizes the image pixels , as the value of pixels can be from 0 to 255.(I Reduced it to 0 to 1.This step was important and it did increase my score in the leader board).\n\nMake model created a model and returned it . The model i used was a simple convnet with 3 Conv2D layers ,1 Dense layer ,2 Max Pooling layer and a Dense layer for showing probabiliy of being the digit (Hence it outputs an array of size 10,probability for each).\n\nOptimizer is the common RMSprop one can try SGD too.Loss function is crosscategorical_crossentropy which is best suited for such conditions and accuracy as metrics to monitor performance of my model.\n\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:01.802754Z","iopub.execute_input":"2021-06-12T20:13:01.803372Z","iopub.status.idle":"2021-06-12T20:13:09.883598Z","shell.execute_reply.started":"2021-06-12T20:13:01.803334Z","shell.execute_reply":"2021-06-12T20:13:09.882681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(20,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    print(len(train_indices))\n    his = model.fit(trainX[train_indices],trainy[train_indices],validation_data=(trainX[val_indices],trainy[val_indices]),epochs=1,batch_size=512)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:09.884935Z","iopub.execute_input":"2021-06-12T20:13:09.885308Z","iopub.status.idle":"2021-06-12T20:13:40.705412Z","shell.execute_reply.started":"2021-06-12T20:13:09.885270Z","shell.execute_reply":"2021-06-12T20:13:40.704576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Instead of running a simple model.fit ,I used Stratified Shuffle Split for 2 reasons:-\n1. It divides the data properly , assume i have 50 images of cat and dogs. And i decided to use 50 for training ,25 for validation and 25 for testing . There is no way to tell if 50 training images are of 25 cats and dog each .Maybe its 40 cats and 10 dogs ,Surely this is a problem as it wont be truly proportional sample. This problem is taken care by Stratified Shuffle Split .\n\n2. I got better score using this method.POINT BLANK.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nplt.plot(score_acc)\nplt.plot(score_val_accuracy)\nplt.axis([0,20,.96,1.05])","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:40.709022Z","iopub.execute_input":"2021-06-12T20:13:40.709324Z","iopub.status.idle":"2021-06-12T20:13:40.894680Z","shell.execute_reply.started":"2021-06-12T20:13:40.709295Z","shell.execute_reply":"2021-06-12T20:13:40.893869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:40.897723Z","iopub.execute_input":"2021-06-12T20:13:40.898098Z","iopub.status.idle":"2021-06-12T20:13:41.907093Z","shell.execute_reply.started":"2021-06-12T20:13:40.898064Z","shell.execute_reply":"2021-06-12T20:13:41.906292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('CONV 2D.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:41.908471Z","iopub.execute_input":"2021-06-12T20:13:41.908832Z","iopub.status.idle":"2021-06-12T20:13:42.279630Z","shell.execute_reply.started":"2021-06-12T20:13:41.908794Z","shell.execute_reply":"2021-06-12T20:13:42.278814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model above gave me a good 99. something accuracy which i was suprised to see as which my normal Nueral network i got 97. something at max.\nI would like the readers who are starting in kaggle to learn more about Convulation Network as it blew me away. I was reluctant to use it as i thought of it as some advanced topic and kept on delaying to learn it .I wish you would not make the same mistake as me.","metadata":{}},{"cell_type":"markdown","source":"AUGMENTATION","metadata":{}},{"cell_type":"markdown","source":"Now i read about data augmentation and this was easy to understand but i was dumbstruck on how to use it and keras documentaion did help me.\nIf you are hearing about Data Augmentation for first time , please go through the link to get an Idea about this and i Promise you will love it.\nhttps://www.analyticsvidhya.com/blog/2020/08/image-augmentation-on-the-fly-using-keras-imagedatagenerator/","metadata":{}},{"cell_type":"markdown","source":"This was just a rough code as i wanted to see the images it generated .Curiousity got the best of me.","metadata":{}},{"cell_type":"code","source":"trainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:42.280827Z","iopub.execute_input":"2021-06-12T20:13:42.281184Z","iopub.status.idle":"2021-06-12T20:13:46.050308Z","shell.execute_reply.started":"2021-06-12T20:13:42.281147Z","shell.execute_reply":"2021-06-12T20:13:46.049526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = image.ImageDataGenerator()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:46.053198Z","iopub.execute_input":"2021-06-12T20:13:46.053447Z","iopub.status.idle":"2021-06-12T20:13:46.058067Z","shell.execute_reply.started":"2021-06-12T20:13:46.053421Z","shell.execute_reply":"2021-06-12T20:13:46.057102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(10,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:13:46.059260Z","iopub.execute_input":"2021-06-12T20:13:46.059621Z","iopub.status.idle":"2021-06-12T20:14:04.027636Z","shell.execute_reply.started":"2021-06-12T20:13:46.059581Z","shell.execute_reply":"2021-06-12T20:14:04.026760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:04.029084Z","iopub.execute_input":"2021-06-12T20:14:04.029424Z","iopub.status.idle":"2021-06-12T20:14:07.931831Z","shell.execute_reply.started":"2021-06-12T20:14:04.029394Z","shell.execute_reply":"2021-06-12T20:14:07.930909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG.csv',index=False)\nmodel.save('DATA AUG.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:07.933328Z","iopub.execute_input":"2021-06-12T20:14:07.938040Z","iopub.status.idle":"2021-06-12T20:14:09.024256Z","shell.execute_reply.started":"2021-06-12T20:14:07.937992Z","shell.execute_reply":"2021-06-12T20:14:09.023496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You will see that i save my files in csv and their names might be a bit confusing but i do this to keep track of what i did to get the results like data aug means i added augmentation to my code of data .It is a little unorthodox but i will make sure to find better way.","metadata":{}},{"cell_type":"markdown","source":"I used GPU for these fittings and it saved me hours of time .Hatsoff to kaggle GPU for this , i would also suggest you to learn how to use it as it will help to make your code execute faster but only if you are training deep Networks.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nplt.plot(score_acc,label='train')\nplt.plot(score_val_accuracy,label='test')\nplt.axis([0,20,.96,1.])\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:09.025548Z","iopub.execute_input":"2021-06-12T20:14:09.025880Z","iopub.status.idle":"2021-06-12T20:14:09.194772Z","shell.execute_reply.started":"2021-06-12T20:14:09.025844Z","shell.execute_reply":"2021-06-12T20:14:09.193853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nplt.plot(score_loss,label='train')\nplt.plot(score_val_loss,label='test')\nplt.axis([0,11,.0,.1])\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:09.196094Z","iopub.execute_input":"2021-06-12T20:14:09.196628Z","iopub.status.idle":"2021-06-12T20:14:09.339507Z","shell.execute_reply.started":"2021-06-12T20:14:09.196586Z","shell.execute_reply":"2021-06-12T20:14:09.338336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some plots to see the performance of my model.","metadata":{}},{"cell_type":"code","source":"\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(20,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:09.340882Z","iopub.execute_input":"2021-06-12T20:14:09.341246Z","iopub.status.idle":"2021-06-12T20:14:42.715331Z","shell.execute_reply.started":"2021-06-12T20:14:09.341206Z","shell.execute_reply":"2021-06-12T20:14:42.714497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size.csv',index=False)\nmodel.save('DATA AUG_size.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:42.716753Z","iopub.execute_input":"2021-06-12T20:14:42.717083Z","iopub.status.idle":"2021-06-12T20:14:43.707400Z","shell.execute_reply.started":"2021-06-12T20:14:42.717054Z","shell.execute_reply":"2021-06-12T20:14:43.706591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here i increase number of splits as data aug do generate more than one image by altering it so i increased the splits,also i could see that it was not reaching upto its true potential.","metadata":{}},{"cell_type":"code","source":"def make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(10,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:14:43.710690Z","iopub.execute_input":"2021-06-12T20:14:43.710955Z","iopub.status.idle":"2021-06-12T20:15:13.651265Z","shell.execute_reply.started":"2021-06-12T20:14:43.710927Z","shell.execute_reply":"2021-06-12T20:15:13.650408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:15:13.653845Z","iopub.execute_input":"2021-06-12T20:15:13.654248Z","iopub.status.idle":"2021-06-12T20:15:17.660408Z","shell.execute_reply.started":"2021-06-12T20:15:13.654205Z","shell.execute_reply":"2021-06-12T20:15:17.659281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch.csv',index=False)\nmodel.save('DATA AUG_size_arch.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:15:17.663719Z","iopub.execute_input":"2021-06-12T20:15:17.664077Z","iopub.status.idle":"2021-06-12T20:15:18.913776Z","shell.execute_reply.started":"2021-06-12T20:15:17.664040Z","shell.execute_reply":"2021-06-12T20:15:18.912958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now that i was in zone and read about VGG 16 please look into it if you have not and i saw the architecture / layers layout they used and i tried to give it a chance . This did certainly help me get past 99.2 accuracy and break my own record.","metadata":{}},{"cell_type":"code","source":"datagen = image.ImageDataGenerator(featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2)\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(10,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:15:18.915170Z","iopub.execute_input":"2021-06-12T20:15:18.915549Z","iopub.status.idle":"2021-06-12T20:17:05.013969Z","shell.execute_reply.started":"2021-06-12T20:15:18.915509Z","shell.execute_reply":"2021-06-12T20:17:05.013220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:17:05.017291Z","iopub.execute_input":"2021-06-12T20:17:05.017606Z","iopub.status.idle":"2021-06-12T20:17:09.048780Z","shell.execute_reply.started":"2021-06-12T20:17:05.017576Z","shell.execute_reply":"2021-06-12T20:17:09.047922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch_custom_gen.csv',index=False)\nmodel.save('DATA AUG_size_arch_custom_gen.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:17:09.050751Z","iopub.execute_input":"2021-06-12T20:17:09.051140Z","iopub.status.idle":"2021-06-12T20:17:10.229731Z","shell.execute_reply.started":"2021-06-12T20:17:09.051085Z","shell.execute_reply":"2021-06-12T20:17:10.228965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I thought to add some customizations to my Image generator , in order to give my score a push . I used the dafault example given in keras documentation .\nhttps://keras.io/api/preprocessing/image/#imagedatagenerator-class","metadata":{}},{"cell_type":"code","source":"datagen = image.ImageDataGenerator(featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2)\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(20,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:17:10.232277Z","iopub.execute_input":"2021-06-12T20:17:10.232652Z","iopub.status.idle":"2021-06-12T20:20:38.689941Z","shell.execute_reply.started":"2021-06-12T20:17:10.232612Z","shell.execute_reply":"2021-06-12T20:20:38.689195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:20:38.693250Z","iopub.execute_input":"2021-06-12T20:20:38.693509Z","iopub.status.idle":"2021-06-12T20:20:42.580853Z","shell.execute_reply.started":"2021-06-12T20:20:38.693481Z","shell.execute_reply":"2021-06-12T20:20:42.580004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch_custom_gen_20.csv',index=False)\nmodel.save('DATA AUG_size_arch_custom_gen_20.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:20:42.582190Z","iopub.execute_input":"2021-06-12T20:20:42.582557Z","iopub.status.idle":"2021-06-12T20:20:43.745276Z","shell.execute_reply.started":"2021-06-12T20:20:42.582518Z","shell.execute_reply":"2021-06-12T20:20:43.744483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I thought it was not running my model enough as it was kind of underfiting so i changed it to 20 splits.","metadata":{}},{"cell_type":"code","source":"datagen = image.ImageDataGenerator(featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2)\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    \n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dropout(.25))\n    model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(20,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:20:43.746498Z","iopub.execute_input":"2021-06-12T20:20:43.746842Z","iopub.status.idle":"2021-06-12T20:24:12.207252Z","shell.execute_reply.started":"2021-06-12T20:20:43.746805Z","shell.execute_reply":"2021-06-12T20:24:12.206472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:24:12.210567Z","iopub.execute_input":"2021-06-12T20:24:12.210825Z","iopub.status.idle":"2021-06-12T20:24:16.932471Z","shell.execute_reply.started":"2021-06-12T20:24:12.210798Z","shell.execute_reply":"2021-06-12T20:24:16.931378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch_custom_gen_20_DROPOUT.csv',index=False)\nmodel.save('DATA AUG_size_arch_custom_gen_20_DROPOUT.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:24:16.936272Z","iopub.execute_input":"2021-06-12T20:24:16.936618Z","iopub.status.idle":"2021-06-12T20:24:18.096630Z","shell.execute_reply.started":"2021-06-12T20:24:16.936582Z","shell.execute_reply":"2021-06-12T20:24:18.095836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now i my model was giving 99.99 something accuracy but it was not performing as good in the scoreboard so i decided to add a dropout layer with probability .\nFor those who heard this term the first time,it is a simple way to prevent overfitting in your model as it randomly selects some nodes and shuts them off , which makes the other node to learn on their own .Hence in a way no node resides on top as each learns the same amount .This link can help with a more better explaintation.\nhttps://machinelearningmastery.com/dropout-for-regularizing-deep-neural-networks/","metadata":{}},{"cell_type":"code","source":"datagen = image.ImageDataGenerator(featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,zoom_range=0.08,shear_range=0.3)\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    \n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dropout(.25))\n    model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(20,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:24:18.099084Z","iopub.execute_input":"2021-06-12T20:24:18.099652Z","iopub.status.idle":"2021-06-12T20:28:21.045699Z","shell.execute_reply.started":"2021-06-12T20:24:18.099607Z","shell.execute_reply":"2021-06-12T20:28:21.044979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:28:21.049058Z","iopub.execute_input":"2021-06-12T20:28:21.049330Z","iopub.status.idle":"2021-06-12T20:28:25.065631Z","shell.execute_reply.started":"2021-06-12T20:28:21.049303Z","shell.execute_reply":"2021-06-12T20:28:25.064888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch_custom_gen_20_DROPOUT_gen.csv',index=False)\nmodel.save('DATA AUG_size_arch_custom_gen_20_DROPOUT_gen.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:28:25.066950Z","iopub.execute_input":"2021-06-12T20:28:25.067308Z","iopub.status.idle":"2021-06-12T20:28:26.255556Z","shell.execute_reply.started":"2021-06-12T20:28:25.067271Z","shell.execute_reply":"2021-06-12T20:28:26.254764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here i added some touch to the image generator like zoom,shear transfomation ,etc .which made it past 99.3 .","metadata":{}},{"cell_type":"code","source":"datagen = image.ImageDataGenerator(featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,zoom_range=0.08,shear_range=0.3)\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    \n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dense(512, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(30,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:28:26.260763Z","iopub.execute_input":"2021-06-12T20:28:26.261033Z","iopub.status.idle":"2021-06-12T20:34:30.852181Z","shell.execute_reply.started":"2021-06-12T20:28:26.261007Z","shell.execute_reply":"2021-06-12T20:34:30.851396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:34:30.855568Z","iopub.execute_input":"2021-06-12T20:34:30.855830Z","iopub.status.idle":"2021-06-12T20:34:35.440315Z","shell.execute_reply.started":"2021-06-12T20:34:30.855801Z","shell.execute_reply":"2021-06-12T20:34:35.439489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch_custom_gen_20_DROPOUT_gen_dense-30.csv',index=False)\nmodel.save('DATA AUG_size_arch_custom_gen_20_DROPOUT_gen_dense-30.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:34:35.443157Z","iopub.execute_input":"2021-06-12T20:34:35.443409Z","iopub.status.idle":"2021-06-12T20:34:36.571230Z","shell.execute_reply.started":"2021-06-12T20:34:35.443383Z","shell.execute_reply":"2021-06-12T20:34:36.570434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here i did two things :-\n1. I increased splits to 30 .\n2. I increased nodes in Dense layer to 512 .\n\nMy model was neither underfitting/overfitting but for some reason i had a feeling that i can do better .  ","metadata":{}},{"cell_type":"code","source":"datagen = image.ImageDataGenerator(featurewise_center=True,\n    featurewise_std_normalization=True,\n    rotation_range=30,\n    width_shift_range=0.25,\n    height_shift_range=0.25,zoom_range=0.08,shear_range=0.3)\ndef make_model():\n    keras.backend.clear_session()\n    model = keras.models.Sequential()\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',input_shape=(28,28,1),padding='same'))\n    model.add(Conv2D(32,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ##\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(64,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    ###\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(Conv2D(128,(3,3),activation='relu',kernel_initializer='he_uniform',padding='same'))\n    model.add(MaxPooling2D((2, 2)))\n    \n    model.add(Flatten())\n    model.add(Dense(512, activation='relu', kernel_initializer='he_uniform'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = 'RMSprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model\ntrainX, trainy,testX = load_data()\ntrainX,testX = normalize(trainX,testX)\nmodel = make_model()\nmodel.summary()\nscore_acc = []\nscore_loss = []\nscore_val_loss = []\nscore_val_accuracy = []\nsss = StratifiedShuffleSplit(30,test_size=.2,random_state=115)\nfor index, (train_indices, val_indices) in enumerate(sss.split(trainX,trainy)):\n    aa = trainX[train_indices]\n    ab = trainy[train_indices]\n    cc = trainX[val_indices]\n    cd = trainy[val_indices]\n    training_batch = datagen.flow(aa, ab, batch_size=512)\n    val_batches = datagen.flow(cc, cd, batch_size=128)\n    his = model.fit_generator(generator=training_batch, steps_per_epoch=512, epochs=1, \n                    validation_data=val_batches, validation_steps=128)\n    score_acc.append(his.history['accuracy'])\n    score_loss.append(his.history['loss'])\n    score_val_loss.append(his.history['val_loss'])\n    score_val_accuracy.append(his.history['val_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:34:36.572543Z","iopub.execute_input":"2021-06-12T20:34:36.572871Z","iopub.status.idle":"2021-06-12T20:40:39.851358Z","shell.execute_reply.started":"2021-06-12T20:34:36.572836Z","shell.execute_reply":"2021-06-12T20:40:39.850581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(trainX,trainy)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:40:39.854513Z","iopub.execute_input":"2021-06-12T20:40:39.854776Z","iopub.status.idle":"2021-06-12T20:40:43.782837Z","shell.execute_reply.started":"2021-06-12T20:40:39.854748Z","shell.execute_reply":"2021-06-12T20:40:43.781971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(testX),axis=1)\npred = pd.DataFrame(pred,columns=['Label'])\nxyz = pd.DataFrame(np.arange(1,testX.shape[0]+1),columns=['ImageId'])\n\npred = pd.concat([xyz,pred],axis=1)\npred.to_csv('DATA AUG_size_arch_custom_gen_20_DROPOUT_gen_dense_gen_30.csv',index=False)\nmodel.save('DATA AUG_size_arch_custom_gen_20_DROPOUT_gen_dense_gen_30.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:40:43.784057Z","iopub.execute_input":"2021-06-12T20:40:43.784412Z","iopub.status.idle":"2021-06-12T20:40:44.966171Z","shell.execute_reply.started":"2021-06-12T20:40:43.784373Z","shell.execute_reply":"2021-06-12T20:40:44.965298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this i just tweaked the geneareator a bit and not much as i was getting good results.","metadata":{}},{"cell_type":"markdown","source":"Now i tried to see where my models were giving different answers , and some images were even difficult for me to understand .\nLOL","metadata":{}},{"cell_type":"code","source":"# ../input/digit-recogniser-my-answers/DATA AUG.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:40:44.967300Z","iopub.execute_input":"2021-06-12T20:40:44.967629Z","iopub.status.idle":"2021-06-12T20:40:44.974577Z","shell.execute_reply.started":"2021-06-12T20:40:44.967595Z","shell.execute_reply":"2021-06-12T20:40:44.973717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfiles = []\nfor dirname, _, filenames in os.walk('/kaggle/working'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        if 'csv' in filename:\n            files.append(os.path.join(dirname, filename))\n       \n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:40:44.975955Z","iopub.execute_input":"2021-06-12T20:40:44.976375Z","iopub.status.idle":"2021-06-12T20:40:44.987015Z","shell.execute_reply.started":"2021-06-12T20:40:44.976337Z","shell.execute_reply":"2021-06-12T20:40:44.985736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_data = []\nfor i in files:\n    aa = pd.read_csv(i,index_col=False).values[:,1]\n    print(aa)\n    label_data.append(aa)\nlabel_data = np.array(label_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:41:00.806097Z","iopub.execute_input":"2021-06-12T20:41:00.806465Z","iopub.status.idle":"2021-06-12T20:41:00.866976Z","shell.execute_reply.started":"2021-06-12T20:41:00.806432Z","shell.execute_reply":"2021-06-12T20:41:00.866070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(label_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:41:03.308221Z","iopub.execute_input":"2021-06-12T20:41:03.308669Z","iopub.status.idle":"2021-06-12T20:41:03.345700Z","shell.execute_reply.started":"2021-06-12T20:41:03.308629Z","shell.execute_reply":"2021-06-12T20:41:03.344617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(label_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:41:23.889543Z","iopub.execute_input":"2021-06-12T20:41:23.889870Z","iopub.status.idle":"2021-06-12T20:41:23.895863Z","shell.execute_reply.started":"2021-06-12T20:41:23.889841Z","shell.execute_reply":"2021-06-12T20:41:23.894946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"err = []\nfor i,a in enumerate(zip(label_data[0,:],label_data[1,:],label_data[2,:],label_data[3,:],label_data[4,:],label_data[5,:],label_data[6,:],label_data[7,:],label_data[8,:],label_data[9,:])):\n    s = a[0]\n    for b in a:\n        if b!=s:\n            err.append(i)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:41:51.984066Z","iopub.execute_input":"2021-06-12T20:41:51.984419Z","iopub.status.idle":"2021-06-12T20:41:52.080556Z","shell.execute_reply.started":"2021-06-12T20:41:51.984387Z","shell.execute_reply":"2021-06-12T20:41:52.079642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(testX[165,:].reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:41:53.949066Z","iopub.execute_input":"2021-06-12T20:41:53.949407Z","iopub.status.idle":"2021-06-12T20:41:54.091326Z","shell.execute_reply.started":"2021-06-12T20:41:53.949369Z","shell.execute_reply":"2021-06-12T20:41:54.090432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(err))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:41:55.184824Z","iopub.execute_input":"2021-06-12T20:41:55.185174Z","iopub.status.idle":"2021-06-12T20:41:55.190533Z","shell.execute_reply.started":"2021-06-12T20:41:55.185133Z","shell.execute_reply":"2021-06-12T20:41:55.189675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in err[:20]:\n    print(i,end=' ')\n    for j in range(10):\n        print(label_data[j,i],end=' ')\n    print()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:42:28.241868Z","iopub.execute_input":"2021-06-12T20:42:28.242219Z","iopub.status.idle":"2021-06-12T20:42:28.264931Z","shell.execute_reply.started":"2021-06-12T20:42:28.242186Z","shell.execute_reply":"2021-06-12T20:42:28.264117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(testX[165,:].reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:42:31.628896Z","iopub.execute_input":"2021-06-12T20:42:31.629466Z","iopub.status.idle":"2021-06-12T20:42:31.775666Z","shell.execute_reply.started":"2021-06-12T20:42:31.629416Z","shell.execute_reply":"2021-06-12T20:42:31.774609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_pred = pd.DataFrame(label_data).mode(axis=0).iloc[0]\nstacked_pred = stacked_pred.apply(lambda x:int(x))\nstacked_pred = pd.DataFrame({'Label': stacked_pred})\nxyz = pd.DataFrame(np.arange(1,stacked_pred.shape[0]+1),columns=['ImageId'])\n\nstacked_pred = pd.concat([xyz,stacked_pred],axis=1)\nstacked_pred.to_csv('STACK_OF_Ten.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T20:42:39.524032Z","iopub.execute_input":"2021-06-12T20:42:39.524413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At Last i used stacked my different result that i had and used simple mode to decide my answer .\nMy submissions were completed at this point but i have good feeling about this predictions.","metadata":{}},{"cell_type":"markdown","source":"This marks the end of the notebook , hope it helps in your journey and hope we meet again soon .\n\nTHANK YOU FOR READING ....","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}