{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"1. Introduction\n2. Data preparation  \n    2.1. Load data  \n    2.2. Check for null and missing values  \n    2.3. Normalization  \n    2.4. Reshape  \n    2.5. Label encoding  \n    2.6. Split training and validation set  \n3. CNN  \n    3.1. Define the model  \n    3.2. Set the optimizer and aneealer  \n    3.3. Data aumentation  \n4. Evaluate the model  \n    4.1. Training and validation curves  \n    4.2. Confusion matrix  \n5. Prediction and submittion  \n    5.1. Predict and Submit results  ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true}},{"cell_type":"markdown","source":"# 1. Introduction\nThis is a 5 layers Sequential Convolutional Neural Network for ditis recognition trained on MNIST dataset. I choosed to build it with keras API which is very intuitive. Firstly, I will prepare the data then i will focus on the CNN modeling and evaluation.","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D,BatchNormalization\nfrom keras.optimizers import Adam, RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\nsns.set(style='white', context='notebook', palette='deep')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data preparation\n## 2.1. Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2. Check for null and missing values","metadata":{}},{"cell_type":"code","source":"Y_train = train['label']\n\nX_train = train.drop(labels=['label'], axis = 1)\n\ng = sns.countplot(Y_train)\nY_train.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.isnull().any().describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().any().describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.3. Normalization","metadata":{}},{"cell_type":"code","source":"X_train = X_train / 255.0\ntest = test / 255.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.4. Reshape","metadata":{}},{"cell_type":"code","source":"X_train = X_train.values.reshape(-1, 28, 28, 1)\ntest = test.values.reshape(-1, 28, 28, 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.5. Label encoding","metadata":{}},{"cell_type":"code","source":"Y_train = to_categorical(Y_train, num_classes=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.6. Split training and validataion set","metadata":{}},{"cell_type":"code","source":"random_seed = 2\n\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = plt.imshow(X_train[0][:,:,0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. CNN\n## 3.1. Define the model","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(32,kernel_size=3,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.4))\n\nmodel.add(Conv2D(64,kernel_size=3,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(64,kernel_size=3,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.4))\n\nmodel.add(Flatten())\nmodel.add(Dense(128, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.4))\nmodel.add(Dense(10, activation='softmax'))\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2. Set the optimizer and annealer","metadata":{}},{"cell_type":"code","source":"optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n# optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = optimizer, loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)\n\nepochs = 50\nbatch_size = 10","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.3. Data augmentation","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False,  # randomly flip images\n)\n\ndatagen.fit(X_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Evaluate the model\n## 4.1. Training and validation curves","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict results\nresults = model.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}