{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Content\n1. [Python Libraries](#1)\n1. [Loading the Data Set](#2)\n1. [Normalization, Reshape, Label Encoding, Batch Size and Epochs](#3)\n1. [Train - Test Split](#4)\n1. [Creating CNN Model](#5)\n1. [Compile Model](#6)\n1. [Data Augmentation](#7)\n1. [Training](#8)\n1. [Evaluation](#9)\n1. [Prediction on Test Data Set](#10)\n1. [Submission](#11)","metadata":{}},{"cell_type":"markdown","source":"<a id = '1' ></a>\n## Python Libraries","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# import some additional tools\nimport matplotlib.pyplot as plt      # MATLAB like plotting routines\nimport random                        # for generating random numbers\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, Flatten\nfrom tensorflow.keras.layers import BatchNormalization\nfrom keras.models import Sequential  # Model type to be used\nfrom keras.layers.core import Dense, Dropout, Activation # Types of layers to be used in our model\nfrom keras.utils import np_utils\nfrom sklearn.model_selection import train_test_split\nimport warnings\nwarnings.filterwarnings('ignore')\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-25T22:20:42.776342Z","iopub.execute_input":"2022-11-25T22:20:42.777304Z","iopub.status.idle":"2022-11-25T22:20:42.787696Z","shell.execute_reply.started":"2022-11-25T22:20:42.777244Z","shell.execute_reply":"2022-11-25T22:20:42.786524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '2' ></a>\n## Loading the Data Set","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:42.789842Z","iopub.execute_input":"2022-11-25T22:20:42.790225Z","iopub.status.idle":"2022-11-25T22:20:45.913475Z","shell.execute_reply.started":"2022-11-25T22:20:42.790167Z","shell.execute_reply":"2022-11-25T22:20:45.912432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train Shape:\", train.shape)\nprint(\"Test Shape:\",test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:45.918384Z","iopub.execute_input":"2022-11-25T22:20:45.919139Z","iopub.status.idle":"2022-11-25T22:20:45.929429Z","shell.execute_reply.started":"2022-11-25T22:20:45.919102Z","shell.execute_reply":"2022-11-25T22:20:45.928572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '3' ></a>\n## Normalization, Reshape, Label Encoding, Batch Size and Epochs","metadata":{}},{"cell_type":"code","source":"X_train = (train.iloc[:,1:].values).astype('float32') # all pixel values\nY_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits\nX_test = test.values.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:45.933839Z","iopub.execute_input":"2022-11-25T22:20:45.934680Z","iopub.status.idle":"2022-11-25T22:20:46.060253Z","shell.execute_reply.started":"2022-11-25T22:20:45.934646Z","shell.execute_reply":"2022-11-25T22:20:46.059259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.reshape(X_train.shape[0], 28, 28, 1) #add an additional dimension to represent the single-channel\nX_test = X_test.reshape(X_test.shape[0], 28, 28, 1)\n\nX_train /= 255                              # normalize each value for each pixel for the entire vector for each input\nX_test /= 255\n\nprint(\"Training matrix shape\", X_train.shape)\nprint(\"Testing matrix shape\", X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:46.063018Z","iopub.execute_input":"2022-11-25T22:20:46.063425Z","iopub.status.idle":"2022-11-25T22:20:46.089894Z","shell.execute_reply.started":"2022-11-25T22:20:46.063386Z","shell.execute_reply":"2022-11-25T22:20:46.088847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one-hot format classes\n\nnb_classes = 10 # number of unique digits\nbatch_size = 64\nepochs = 20\n\nY_train = np_utils.to_categorical(Y_train, nb_classes)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:46.092129Z","iopub.execute_input":"2022-11-25T22:20:46.093081Z","iopub.status.idle":"2022-11-25T22:20:46.099443Z","shell.execute_reply.started":"2022-11-25T22:20:46.093034Z","shell.execute_reply":"2022-11-25T22:20:46.098483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '4' ></a>\n## Train - Test Split","metadata":{}},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:46.100980Z","iopub.execute_input":"2022-11-25T22:20:46.101370Z","iopub.status.idle":"2022-11-25T22:20:46.415579Z","shell.execute_reply.started":"2022-11-25T22:20:46.101333Z","shell.execute_reply":"2022-11-25T22:20:46.414600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '5' ></a>\n## Creating CNN Model","metadata":{}},{"cell_type":"code","source":"model = Sequential()                                 # Linear stacking of layers\n\n# Convolution Layer 1\nmodel.add(Conv2D(32, (3, 3), input_shape=(28,28,1))) # 32 different 3x3 kernels -- so 32 feature maps\nmodel.add(BatchNormalization(axis=-1))               # normalize each feature map before activation\nconvLayer01 = Activation('relu')                     # activation\nmodel.add(convLayer01)\n\n# Convolution Layer 2\nmodel.add(Conv2D(32, (3, 3)))                        # 32 different 3x3 kernels -- so 32 feature maps\nmodel.add(BatchNormalization(axis=-1))               # normalize each feature map before activation\nmodel.add(Activation('relu'))                        # activation\nconvLayer02 = MaxPooling2D(pool_size=(2,2))          # Pool the max values over a 2x2 kernel\nmodel.add(convLayer02)\n\n# Convolution Layer 3\nmodel.add(Conv2D(64,(3, 3)))                         # 64 different 3x3 kernels -- so 64 feature maps\nmodel.add(BatchNormalization(axis=-1))               # normalize each feature map before activation\nconvLayer03 = Activation('relu')                     # activation\nmodel.add(convLayer03)\n\n# Convolution Layer 4\nmodel.add(Conv2D(64, (3, 3)))                        # 64 different 3x3 kernels -- so 64 feature maps\nmodel.add(BatchNormalization(axis=-1))               # normalize each feature map before activation\nmodel.add(Activation('relu'))                        # activation\nconvLayer04 = MaxPooling2D(pool_size=(2,2))          # Pool the max values over a 2x2 kernel\nmodel.add(convLayer04)\nmodel.add(Flatten())                                 # Flatten final 4x4x64 output matrix into a 1024-length vector\n\n# Fully Connected Layer 5\nmodel.add(Dense(512))                                # 512 FCN nodes\nmodel.add(BatchNormalization())                      # normalization\nmodel.add(Activation('relu'))                        # activation\n\n# Fully Connected Layer 6                       \nmodel.add(Dropout(0.2))                              # 20% dropout of randomly selected nodes\nmodel.add(Dense(10))                                 # final 10 FCN nodes\nmodel.add(Activation('softmax'))                     # softmax activation","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-11-25T22:20:46.417084Z","iopub.execute_input":"2022-11-25T22:20:46.417695Z","iopub.status.idle":"2022-11-25T22:20:49.244354Z","shell.execute_reply.started":"2022-11-25T22:20:46.417655Z","shell.execute_reply":"2022-11-25T22:20:49.242266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:49.245787Z","iopub.execute_input":"2022-11-25T22:20:49.246119Z","iopub.status.idle":"2022-11-25T22:20:49.253391Z","shell.execute_reply.started":"2022-11-25T22:20:49.246084Z","shell.execute_reply":"2022-11-25T22:20:49.252352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '6' ></a>\n## Compile Model","metadata":{}},{"cell_type":"code","source":"# we'll use Adam Optimizer\n\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:49.254835Z","iopub.execute_input":"2022-11-25T22:20:49.255419Z","iopub.status.idle":"2022-11-25T22:20:49.271289Z","shell.execute_reply.started":"2022-11-25T22:20:49.255383Z","shell.execute_reply":"2022-11-25T22:20:49.270237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '7' ></a>\n## Data Augmentation","metadata":{}},{"cell_type":"code","source":"# data augmentation prevents overfitting by slightly changing the data randomly\n# Keras has a great built-in feature to do automatic augmentation\n\ngen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n                         height_shift_range=0.08, zoom_range=0.08)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:49.275276Z","iopub.execute_input":"2022-11-25T22:20:49.275525Z","iopub.status.idle":"2022-11-25T22:20:49.280931Z","shell.execute_reply.started":"2022-11-25T22:20:49.275502Z","shell.execute_reply":"2022-11-25T22:20:49.279865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can then feed our augmented data in batches\n# Besides loss function considerations as before, this method actually results in significant memory savings\n# because we are actually LOADING the data into the network in batches before processing each batch\n\n# Before the data was all loaded into memory, but then processed in batches.\n\ntrain_generator = gen.flow(X_train, Y_train, batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:49.282386Z","iopub.execute_input":"2022-11-25T22:20:49.282810Z","iopub.status.idle":"2022-11-25T22:20:49.290184Z","shell.execute_reply.started":"2022-11-25T22:20:49.282776Z","shell.execute_reply":"2022-11-25T22:20:49.289331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '8' ></a>\n## Training","metadata":{}},{"cell_type":"code","source":"# We can now train our model which is fed data by our batch loader\n# Steps per epoch should always be total size of the set divided by the batch size\n\n# SIGNIFICANT MEMORY SAVINGS (important for larger, deeper networks)\n\nmodel.fit_generator(train_generator, steps_per_epoch=X_train.shape[0] // batch_size, epochs = epochs, verbose=1, \n                    validation_data = (X_val,Y_val), validation_steps=X_val.shape[0] // batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:20:49.291639Z","iopub.execute_input":"2022-11-25T22:20:49.293136Z","iopub.status.idle":"2022-11-25T22:26:25.782480Z","shell.execute_reply.started":"2022-11-25T22:20:49.293109Z","shell.execute_reply":"2022-11-25T22:26:25.781337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '9' ></a>\n## Evaluation","metadata":{}},{"cell_type":"code","source":"score = model.evaluate(X_val, Y_val)\nprint('Val accuracy:', score[1])","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:26:25.783883Z","iopub.execute_input":"2022-11-25T22:26:25.784370Z","iopub.status.idle":"2022-11-25T22:26:26.246152Z","shell.execute_reply.started":"2022-11-25T22:26:25.784332Z","shell.execute_reply":"2022-11-25T22:26:26.245265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '10' ></a>\n## Prediction on Test Data Set","metadata":{}},{"cell_type":"code","source":"# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:26:26.247637Z","iopub.execute_input":"2022-11-25T22:26:26.247973Z","iopub.status.idle":"2022-11-25T22:26:26.742696Z","shell.execute_reply.started":"2022-11-25T22:26:26.247939Z","shell.execute_reply":"2022-11-25T22:26:26.741693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_classes[:5]","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:26:26.746947Z","iopub.execute_input":"2022-11-25T22:26:26.747551Z","iopub.status.idle":"2022-11-25T22:26:26.757399Z","shell.execute_reply.started":"2022-11-25T22:26:26.747514Z","shell.execute_reply":"2022-11-25T22:26:26.754677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = '11' ></a>\n## Submission","metadata":{}},{"cell_type":"code","source":"prediction = model.predict(X_test)\npredicted_classes = np.argmax(prediction,axis=1)\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predicted_classes)+1)),\n                         \"Label\": predicted_classes})\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T22:26:26.759594Z","iopub.execute_input":"2022-11-25T22:26:26.759947Z","iopub.status.idle":"2022-11-25T22:26:28.602923Z","shell.execute_reply.started":"2022-11-25T22:26:26.759909Z","shell.execute_reply":"2022-11-25T22:26:28.601988Z"},"trusted":true},"execution_count":null,"outputs":[]}]}