{"cells":[{"metadata":{"_uuid":"a0afad01e92f1401ee91cb2b33e06f8a085eeeb4"},"cell_type":"markdown","source":"# Digit recognizer with Keras\nCode used from \n* kaggle Deep Learning track Lesson 8: [Dropout and Strides For Larger Models](https://www.kaggle.com/dansbecker/dropout-and-strides-for-larger-models)\n* [Introduction to CNN Keras - 0.997 (top 6%)](https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6)\n* [Welcome to deep learning (CNN 99%)](https://www.kaggle.com/toregil/welcome-to-deep-learning-cnn-99/notebook)"},{"metadata":{"_uuid":"3fae6333ed2317813e29cad2739f895cd4e4170e"},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nimport keras\nfrom keras.utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import BatchNormalization, Conv2D, Dense, Dropout, Flatten, MaxPool2D\nfrom keras.optimizers import RMSprop\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator","execution_count":66,"outputs":[]},{"metadata":{"_uuid":"baa88ebc0b3103ae91a538cf51b0f06bb0c28fb9"},"cell_type":"markdown","source":"## Parameters"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"TRAIN_FILE = \"../input/train.csv\"\nTEST_FILE = \"../input/test.csv\"\n\nIMG_ROWS, IMG_COLS = 28, 28\nNUM_CLASSES = 10\n\nTEST_SIZE = 0.1\n\nFILTERS_1 = 32\nKERNEL_SIZE_1 = (5, 5)\nFILTERS_2 = 64\nKERNEL_SIZE_2 = (3, 3)\nSTRIDES = 2\nACTIVATION_CONV2D = 'relu'\n\nUNITS_DENSE = 256\nACTIVATION_DENSE = 'relu'\n\nLOSS = keras.losses.categorical_crossentropy\nOPTIMIZER = 'adam'\n#OPTIMIZER = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\nMETRICS = ['accuracy']\n\nBATCH_SIZE = 64\nEPOCHS = 30","execution_count":67,"outputs":[]},{"metadata":{"_uuid":"9999a9f663be0e57cb5d0894d88cb947d05f44bb"},"cell_type":"markdown","source":"## Functions"},{"metadata":{"trusted":true,"_uuid":"5c1309827abe1a33930ea1d03980160c0106a806","collapsed":true},"cell_type":"code","source":"def data_prep_train(train):\n    Y_train = to_categorical(train.label, \n                             num_classes = NUM_CLASSES)\n\n    num_images = train.shape[0]\n    X_as_array = train.values[:,1:]\n    X_shaped_array = X_as_array.reshape(num_images, IMG_ROWS, IMG_ROWS, 1)\n    X_train = X_shaped_array / 255.0\n    return X_train, Y_train\n\ndef data_prep_test(test):\n\n    num_images = test.shape[0]\n    test_as_array = test.values\n    test_shaped_array = test_as_array.reshape(num_images, IMG_ROWS, IMG_ROWS, 1)\n    test = test_shaped_array / 255.0\n    return test\n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":68,"outputs":[]},{"metadata":{"_uuid":"8a27e887e1286831bbf3c89f1d29019bfceeaf67"},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true,"_uuid":"87e5ee82da3388cb4cca8d11e0a41ca1ff563847","collapsed":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_FILE)\ntest = pd.read_csv(TEST_FILE)","execution_count":69,"outputs":[]},{"metadata":{"_uuid":"5a4bdc94362e18fc981683899a64c6f698265ff5"},"cell_type":"markdown","source":"## Prepare data"},{"metadata":{"trusted":true,"_uuid":"ae9a19c5b258c28af9eeb8071994f82fbdc9d619","collapsed":true},"cell_type":"code","source":"X_train, Y_train = data_prep_train(train)\n\ntest = data_prep_test(test)\n","execution_count":70,"outputs":[]},{"metadata":{"_uuid":"d3a7d5999ba02ae91e8c79d68221e48f8b485633"},"cell_type":"markdown","source":"## Split training and valdiation set"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fc2c786ddd2ccd7fa7eff3245167d1489f701164"},"cell_type":"code","source":"# Set the random seed\nrandom_seed = 2","execution_count":71,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7c13c0904cbf8e3aa12c66fef8f9013349da1c8a"},"cell_type":"code","source":"# Split the train and the validation set for the fitting\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, \n                                                  test_size = TEST_SIZE, \n                                                  random_state=random_seed)","execution_count":72,"outputs":[]},{"metadata":{"_uuid":"7348e274f1c07df262f42b9385c1fb6c110fa006"},"cell_type":"markdown","source":"## CNN model"},{"metadata":{"trusted":true,"_uuid":"b58b87606e893c32a79180156e4dfaa1e0dffc7d","collapsed":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(filters = FILTERS_1, \n                 kernel_size = KERNEL_SIZE_1,\n                 #strides = STRIDES,\n                 activation = ACTIVATION_CONV2D,\n                 input_shape = (IMG_ROWS, IMG_COLS, 1)))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = FILTERS_1, \n                 kernel_size = KERNEL_SIZE_1,\n                 #strides = STRIDES,\n                 activation = ACTIVATION_CONV2D))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.5))\n\nmodel.add(Conv2D(filters = FILTERS_2, \n                 kernel_size = KERNEL_SIZE_2,\n                 #strides = STRIDES,\n                 activation = ACTIVATION_CONV2D))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = FILTERS_2, \n                 kernel_size = KERNEL_SIZE_2,\n                 #strides = STRIDES,\n                 activation = ACTIVATION_CONV2D))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size = (2, 2), strides = (2, 2)))\nmodel.add(Dropout(0.5))\n\nmodel.add(Flatten())\nmodel.add(Dense(UNITS_DENSE, \n                activation = ACTIVATION_DENSE))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(2*UNITS_DENSE, \n                activation = ACTIVATION_DENSE))\nmodel.add(Dense(NUM_CLASSES, \n                activation = 'softmax'))","execution_count":73,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"08523aa08f330b829de8c60c1cbbd23e06383fe9"},"cell_type":"code","source":"# Compile model \nmodel.compile(loss = LOSS,\n              optimizer = OPTIMIZER,\n              metrics = METRICS)","execution_count":74,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6d027eb1010efa55c7a205b754fac5cb37cbb9d9"},"cell_type":"code","source":"# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":75,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a0ba3968d28da3d8cd86e4c29acb124c7f1808a6"},"cell_type":"code","source":"datagen = ImageDataGenerator(zoom_range = 0.1,\n                            height_shift_range = 0.1,\n                            width_shift_range = 0.1,\n                            rotation_range = 10)\n\ndatagen.fit(X_train)","execution_count":76,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f113dab52ce6ed36515c298bee534c8c44c4318d"},"cell_type":"code","source":"# Fit model\nhistory = model.fit_generator(datagen.flow(X_train, Y_train, batch_size = BATCH_SIZE),\n                              steps_per_epoch = X_train.shape[0] // BATCH_SIZE,\n                              epochs = EPOCHS,\n                              verbose = 1,\n                              callbacks = [learning_rate_reduction],\n                              validation_data = (X_val, Y_val))","execution_count":77,"outputs":[]},{"metadata":{"_uuid":"5fd9fe504908f294373fc402739670a46da714d8"},"cell_type":"markdown","source":"## Training and validation curves"},{"metadata":{"trusted":true,"_uuid":"32330d8c27035c057f8a6cd8ff3d7566cb1aef16","collapsed":true},"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"60c43687cbcb859ffc06a7bca0ee0fdf299e58b5"},"cell_type":"markdown","source":"## Confusion matrix"},{"metadata":{"trusted":true,"_uuid":"073d3ea85de901cf6eb2c83ee02f91ab7568d4ac","collapsed":true},"cell_type":"code","source":"# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(Y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) ","execution_count":36,"outputs":[]},{"metadata":{"_uuid":"728dfa06839c1569585ec8921b1fc97f844d351e"},"cell_type":"markdown","source":"## Predict results\n"},{"metadata":{"trusted":true,"_uuid":"e5f44c1bc3af1f3f2d934f7ba3063ba758642d04","collapsed":true},"cell_type":"code","source":"results = model.predict(test)\n\n# select the index with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":37,"outputs":[]},{"metadata":{"_uuid":"539bbd50500824df53fe2c9b3482f1d1c295d3c3"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2c543f2d53733d7c4271c9ae78c353b1d30f933e"},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","execution_count":38,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"160a0d3cd5fc2f4b78a30b945e9ec3f97ea90862"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9bf8300dd035b180ea68d1fec27b7c5a7c81a696"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}