{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"###############################################################\n# NB: shift + tab HOLD FOR 2 SECONDS!\n###############################################################\n\n\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\nprint('\\n Getting traing dataset...')\ntrain_data = pd.read_csv('../input/digit-recognizer/train.csv')\nprint('Traing data set obtained \\n')\n\nprint('Getting test dataset...')\ntest_data = pd.read_csv('../input/digit-recognizer/test.csv')\nprint('Test data set obtained \\n')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# The first function return the matrix out from the dataframe\ndef return_matrix(train, n=0):\n    import numpy as np # linear algebra\n    import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n    import matplotlib.pyplot as plt\n    \n    if ((n>=0) & (n<=train_data.shape[0])) :\n        img_try = train.iloc[n].values.reshape(28,28)\n    else :\n        print('Insert a n between 0 and '.train_data.shape[0])\n        pass \n    return img_try\n\n# The second function eats dataframe plus an integer number between 0 and 21999\ndef print_matrix(train, test, n=0):\n    import numpy as np # linear algebra\n    import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n    import matplotlib.pyplot as plt\n    \n    if ((n>=0) & (n<=train_data.shape[0])) :\n        mat = return_matrix(train, n)\n    \n        fig = plt.figure(figsize=(5, 5))\n        plt.imshow(mat)\n        plt.title(train_label[n], fontsize=30)\n    else : \n        print('Insert a n between 0 and '.train_data.shape[0])\n        pass\n    \n    return print('Done')\n        \n# The third function eats the training dataframe and spits  features and labels\ndef prepare_train_data(df):\n    import pandas as pd \n    import numpy as np\n    from keras import utils as ku\n    \n    print('Transforming the data...')\n    train_img = df.drop('label', axis=1)\n    train_img = train_img/255 #normalize [0,255] to [0.0 , 1.0]\n    features = train_img.values.reshape(-1,28,28,1)\n    \n    labels = ku.to_categorical(df['label'] ,num_classes=10)\n    print('Transformation done \\n')\n    \n    return features, labels\n\n# The fourth function eats the test dataframe and spits the features\ndef prepare_test_data(df):\n    import pandas as pd \n    import numpy as np\n    from keras import utils as ku\n    \n    print('Transforming the data...')\n    train_img = df\n    train_img = train_img/255 #normalize [0,255] to [0.0 , 1.0]\n    features = train_img.values.reshape(-1,28,28,1)\n    \n    print('Transformation done \\n')\n    \n    return features\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features, labels = prepare_train_data(train_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Importing Keras\n\nWe now need to import keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install --upgrade keras\n\nimport sys\nimport keras\nprint('Keras version:',keras.__version__)\n\nfrom keras import backend as K\n\nfrom keras.preprocessing.image import array_to_img\n\nprint('\\n Done. \\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare the Data\nBefore we can train the model, we need to prepare the data. \n\nWe'll divide the feature values by 255 to normalize them as floating point values between 0 and 1, and we'll split the data so that we can use 70% of it to train the model, and hold back 30% to validate it. \n\nWe need also to define the classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train test split\nfrom sklearn.model_selection import train_test_split\n\nprint('Splitting the dataset... ')\nX_train, X_test, Y_train, Y_test = train_test_split(np.array(features), np.array(labels), test_size=0.3)\nprint('Splitting done \\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let us see how the dataset look like:"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)\nprint(X_train.shape[0])\nplt.figure(figsize=(15,8))\nfor i in range(60):\n    plt.subplot(6,10,i+1)\n    plt.imshow(X_train[i].reshape((28,28)),cmap='binary')\n    plt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" # Creating batches for the CNN\nfrom keras.preprocessing.image import ImageDataGenerator\n\nclassnames = '0 1 2 3 4 5 6 7 8 9'.split()\nbatch_size = 128\n\nprint(\"Getting Data...\")\ndatagen = ImageDataGenerator(rescale=1./255, # normalize pixel values\n                             validation_split=0.3) # hold back 30% of the images for validation\n        # Generate batches of tensor image data with real-time data augmentation.\n        # The data will be looped over (in batches).\n\n\nprint(\"Preparing training dataset...\")\ntrain_generator = datagen.flow(X_train, Y_train, batch_size=batch_size)\nprint(\"Training dataset done. \\n\")\n\nprint(\"Preparing validation dataset...\")\nvalidation_generator = datagen.flow(X_test, Y_test, batch_size=batch_size)\nprint(\"Test dataset done. \\n\")\n\nprint('Done. \\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating the CNN Model\n\nWe now need to create our Convolutional Neural Network model. In order to do so, we need to choose the convolution and poolying layers. \n\nWe will thus:\n1. Define the model as a sequential layers\n2. Introduce the Convolutions\n    - they are importan to create a feature map\n3. Introduce the Poolings\n    - MaxPooling is used to reduce dimensionality. \n        In MaxPooling, the output value is just the maximum of the input values in each patch \n        (for ex. The maximum pixel in a span of 3 pixels).\n4. Flatten the data in order to have a np.array to feed the NN\n\n\nIn particoular, we will apply\n1. 16 3x3 Convolution matrix with activation='relu', i.e. a Rectified Linear Unit (ReLU) activation.\n2. A 2x2 MaxPooling\n3. 32 3x3 Convolution matrix with activation='relu', i.e. a Rectified Linear Unit (ReLU) activation.\n4. A 2x2 MaxPooling\n5. 64 3x3 Convolution matrix with activation='relu', i.e. a Rectified Linear Unit (ReLU) activation.\n6. A 2x2 MaxPooling\n7. 64 3x3 Convolution matrix with activation='relu', i.e. a Rectified Linear Unit (ReLU) activation.\n8. A 2x2 MaxPooling\n\nAs a optimaizer, we decided to use the ADAM (ADAptive Moment estimation) optimization algorithm, that is an extension to stochastic gradient descent that has recently seen broader adoption for deep learning applications in computer vision and natural language processing. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define a CNN classifier network\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Flatten, Dense\nfrom keras import optimizers\n\n# Define the model as a sequence of layers\nmodel = Sequential() # Sequential() : Linear stack of layers.\n\n## Add some convolutional layers to extract features = Feature map\nmodel.add(Conv2D(16, (3, 3), padding = 'same', activation = 'relu', input_shape = (28, 28, 1)))\nmodel.add(MaxPooling2D((2, 2)))\n\nmodel.add(Conv2D(32, (3, 3), padding = 'same', activation = 'relu', input_shape = (28, 28, 1)))\nmodel.add(MaxPooling2D((2, 2)))\n\nmodel.add(Conv2D(64, (3, 3), padding = 'same', activation = 'relu'))\nmodel.add(MaxPooling2D((2, 2)))\n\nmodel.add(Conv2D(64, (3, 3), padding = 'same', activation = 'relu'))\nmodel.add(MaxPooling2D((2, 2)))\n\n\n# Now we'll flatten the feature maps \nmodel.add(Flatten())\n\n# and generate an output layer with a predicted probability for each class\nmodel.add(Dense(len(labels[0]), activation='softmax'))\n    # The softmax activation function is used for multi-class classifiers\n\n# We'll use the ADAM optimizer\nopt = optimizers.Adam(lr=0.001)\n\n# With the layers defined, we can now compile the model for categorical (multi-class) classification\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=opt,\n              metrics=['accuracy'])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model over 5 epochs\nnum_epochs = 10\nhistory = model.fit_generator(\n    train_generator,\n    steps_per_epoch = (X_train.shape[0] / batch_size) ,\n    validation_data = validation_generator, \n    epochs = num_epochs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### View the Loss History\n\nWe tracked average training and validation loss and accuracy history for each epoch. \n\nWe can plot these to verify that loss reduced as the model was trained while accuracy increased, and to detect *over-fitting* (which is indicated by a continued drop in training loss after validation loss has levelled out or started to increase)."},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nepoch_nums = range(1,num_epochs+1)\ntraining_loss = history.history[\"loss\"]\nvalidation_loss = history.history[\"val_loss\"]\ntrain_acc = history.history[\"accuracy\"]\nval_acc = history.history['val_accuracy']\n\nplt.figure(figsize=(13,5))\n\nplt.subplot(1,2,1)\nplt.plot(epoch_nums, training_loss)\nplt.plot(epoch_nums, validation_loss)\nplt.xlabel('epoch')\nplt.ylabel('loss')\nplt.legend(['training', 'validation'], loc='upper right')\n\nplt.subplot(1,2,2)\nplt.plot(epoch_nums, train_acc)\nplt.plot(epoch_nums, val_acc)\nplt.xlabel('epoch')\nplt.ylabel('Accuracy')\nplt.legend(['training', 'validation'], loc='lower right')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Confusion Matrix\n\nWe now plot the confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n%matplotlib inline\n\n\nclasses ='0 1 2 3 4 5 6 7 8 9'.split()\nprint(\"Generating predictions from validation data...\")\n# Get the image and label arrays for the first batch of validation data\nx_test = validation_generator[0][0]\ny_test = validation_generator[0][1]\n\n# Use the moedl to predict the class\nclass_probabilities = model.predict(x_test)\n\n# The model returns a probability value for each class\n# The one with the highest probability is the predicted class\npredictions = np.argmax(class_probabilities, axis=1)\n\n# The actual labels are hot encoded (e.g. [0 1 0], so get the one with the value 1\ntrue_labels = np.argmax(y_test, axis=1)\n\n\n# Plot the confusion matrix\ncm = confusion_matrix(true_labels, predictions)\ntick_marks = np.arange(len(classes))\n\n\ndf_cm = pd.DataFrame(cm, index = [i for i in \"0123456789\"], columns = [i for i in \"0123456789\"])\nplt.figure(figsize = (7,7))\nsns.heatmap(df_cm, annot=True, cmap=plt.cm.Blues)\nplt.xlabel(\"Predicted Class\")\nplt.ylabel(\"True Class\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predictions\n\nWe now use the trained CNN to predict the data."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_features = prepare_test_data(test_data)\n\nprint('Starting predictions...')\nfinal = model.predict(test_features)\nfinal = np.argmax(final,axis = 1)\nfinal_df = pd.Series(final, name=\"Label\")\nprint('Predictions done. \\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We now define a function to plot the test image with the predicted label"},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_pred_imag(test_data, final, n=0):\n    import matplotlib.pyplot as plt\n    import pandas as pd\n    import numpy as np\n    \n    if ((n>=0) & (n<=test_data.shape[0])):\n        mat = return_matrix(test_data, n)\n        \n        plt.imshow(mat)\n        plt.title(final[n], fontsize=30)\n        plt.show()\n    else :\n        print('Insert a number between 0 and '.test_data.shape[0])\n        pass\n    \n    return print('Done. \\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_pred_imag(test_data, final, 99)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we plot the first 60 unknown images with their predicted label."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nfor i in range(60):\n    plt.subplot(6,10,i+1)\n    mat = return_matrix(test_data, i)\n    plt.imshow(mat)\n    plt.title(final[i], fontsize=15)\n    plt.axis(\"off\")\n\nplt.subplots_adjust(hspace=1.5)\nplt.subplots_adjust(wspace=0.3)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}