{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Classification of the MNIST dataset\n\n# 1. Introduction\n\nThis notebook will demonstrate how to train a neural network to recognize images of handwritten digits using TensorFlow.  While there are many [MNIST](http://yann.lecun.com/exdb/mnist/) computer vision tutorials that cover the basics, this guide will focus on developing a self contained model that can handle preprocessing.  \nThe contents of this notebook include: \n* Data exploration\n* Data preprocessing\n* Building a Convolutional Neural Network\n* Model training\n* Evaluation of results\n* Predicting outcomes with test data","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-13T17:27:44.004141Z","iopub.execute_input":"2021-10-13T17:27:44.004662Z","iopub.status.idle":"2021-10-13T17:27:44.021499Z","shell.execute_reply.started":"2021-10-13T17:27:44.004543Z","shell.execute_reply":"2021-10-13T17:27:44.020309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.1 Import libraries and verify versions","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential \nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom platform import python_version\n\nprint('Tensorflow Version:', tf.__version__)\nprint('Python Version:', python_version())","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:44.023185Z","iopub.execute_input":"2021-10-13T17:27:44.023672Z","iopub.status.idle":"2021-10-13T17:27:49.953138Z","shell.execute_reply.started":"2021-10-13T17:27:44.023636Z","shell.execute_reply":"2021-10-13T17:27:49.952229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Load the training .csv file and explore the data","metadata":{}},{"cell_type":"code","source":"path = '../input/digit-recognizer/train.csv'\ndf = pd.read_csv(path)\n\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:49.956453Z","iopub.execute_input":"2021-10-13T17:27:49.956763Z","iopub.status.idle":"2021-10-13T17:27:54.067354Z","shell.execute_reply.started":"2021-10-13T17:27:49.956736Z","shell.execute_reply":"2021-10-13T17:27:54.066563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configure random state\nRANDOM_SEED = 123","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:54.071478Z","iopub.execute_input":"2021-10-13T17:27:54.073624Z","iopub.status.idle":"2021-10-13T17:27:54.079074Z","shell.execute_reply.started":"2021-10-13T17:27:54.073568Z","shell.execute_reply":"2021-10-13T17:27:54.078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Isolate the label column from the pixel data.","metadata":{}},{"cell_type":"code","source":"y_labels = df.pop('label')\nx_pixels = df","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:54.0839Z","iopub.execute_input":"2021-10-13T17:27:54.086825Z","iopub.status.idle":"2021-10-13T17:27:54.177682Z","shell.execute_reply.started":"2021-10-13T17:27:54.08678Z","shell.execute_reply":"2021-10-13T17:27:54.176738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.4 Review the labels to confirm the number of unique values.","metadata":{}},{"cell_type":"code","source":"unique_labels = np.unique(y_labels)\nNUM_LABELS = (len(unique_labels))\nprint(\"Labels: \", unique_labels)\nprint(\"No. Labels: \", NUM_LABELS)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:54.182188Z","iopub.execute_input":"2021-10-13T17:27:54.184337Z","iopub.status.idle":"2021-10-13T17:27:54.19716Z","shell.execute_reply.started":"2021-10-13T17:27:54.184293Z","shell.execute_reply":"2021-10-13T17:27:54.196197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The training data has 42,000 images.  Let's examine one of them. ","metadata":{}},{"cell_type":"code","source":"plt.figure()\n# Reshape the array to 28x28 dimensions\nplt.imshow(np.array(x_pixels.iloc[0]).reshape(28, 28)) \nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:54.202414Z","iopub.execute_input":"2021-10-13T17:27:54.204466Z","iopub.status.idle":"2021-10-13T17:27:54.434756Z","shell.execute_reply.started":"2021-10-13T17:27:54.204424Z","shell.execute_reply":"2021-10-13T17:27:54.433942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The colorbar shows the pixel values are within the grayscale range of 0 to 255.  These values will need to be scaled to a range of 0 to 1 which will be covered later in this guide.\nLet's visualize some more images, this time matched with their labels.   ","metadata":{}},{"cell_type":"code","source":"plt.figure()\nfor i in range(8): \n    ax = plt.subplot(2, 4, i + 1)\n    plt.imshow(np.array(x_pixels.iloc[i]).reshape(28, 28))\n    plt.title(y_labels[i])\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:54.436579Z","iopub.execute_input":"2021-10-13T17:27:54.436943Z","iopub.status.idle":"2021-10-13T17:27:54.924453Z","shell.execute_reply.started":"2021-10-13T17:27:54.436906Z","shell.execute_reply":"2021-10-13T17:27:54.923618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With the data verified, it's time to create Datasets for training and validation.","metadata":{}},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(x_pixels, y_labels, test_size=0.2, random_state=RANDOM_SEED)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:54.925983Z","iopub.execute_input":"2021-10-13T17:27:54.926402Z","iopub.status.idle":"2021-10-13T17:27:55.176527Z","shell.execute_reply.started":"2021-10-13T17:27:54.926344Z","shell.execute_reply":"2021-10-13T17:27:55.175636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For this split 80% of the images will be used for training, and 20% for validation.","metadata":{}},{"cell_type":"code","source":"print(x_train.shape)\nprint(x_val.shape)\nprint(y_train.shape)\nprint(y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:55.177745Z","iopub.execute_input":"2021-10-13T17:27:55.178102Z","iopub.status.idle":"2021-10-13T17:27:55.182899Z","shell.execute_reply.started":"2021-10-13T17:27:55.178068Z","shell.execute_reply":"2021-10-13T17:27:55.182076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Create Datasets from the training and validation data\nThe Dataset object is a Python iterable that can be configured for improved performance.  Datasets are used to build efficient input pipelines in TensorFlow.","metadata":{}},{"cell_type":"code","source":"train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train))\nval_ds = tf.data.Dataset.from_tensor_slices((x_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:55.184033Z","iopub.execute_input":"2021-10-13T17:27:55.184412Z","iopub.status.idle":"2021-10-13T17:27:57.679266Z","shell.execute_reply.started":"2021-10-13T17:27:55.184376Z","shell.execute_reply":"2021-10-13T17:27:57.678442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.1 The following code shows 4 different Dataset methods:\n* *Cache()*: Keeps the Dataset elements in memory to be used in later iterations.  \n* *Shuffle()*: Randomly shuffles the elements of the Dataset so each batch can reflect the overall distribution.  This reduces overfitting and variance when training.\n* *Batch()*: Packs the elements of the Dataset into batches.  This determines the number of samples processed during an iteration. \n* *Prefetch()*: Reduces time by overlapping data preprocessing and model execution of a training step.","metadata":{}},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nBUFFER_SIZE = 1000 # Shuffle buffer\nBATCH_SIZE = 32\n\ndef configure_dataset(dataset, shuffle=False, test=False):\n    # Configure the tf dataset for cache, shuffle, batch, and prefetch\n    if shuffle:\n        dataset = dataset.cache()\\\n                        .shuffle(BUFFER_SIZE, seed=RANDOM_SEED, reshuffle_each_iteration=True)\\\n                        .batch(BATCH_SIZE, drop_remainder=True).prefetch(AUTOTUNE)\n    elif test:\n        dataset = dataset.cache()\\\n                        .batch(BATCH_SIZE, drop_remainder=False).prefetch(AUTOTUNE)\n    else:\n        dataset = dataset.cache()\\\n                        .batch(BATCH_SIZE, drop_remainder=True).prefetch(AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:57.680588Z","iopub.execute_input":"2021-10-13T17:27:57.680941Z","iopub.status.idle":"2021-10-13T17:27:57.687872Z","shell.execute_reply.started":"2021-10-13T17:27:57.680905Z","shell.execute_reply":"2021-10-13T17:27:57.686618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = configure_dataset(train_ds, shuffle=True)\nval_ds = configure_dataset(val_ds)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:57.689284Z","iopub.execute_input":"2021-10-13T17:27:57.689771Z","iopub.status.idle":"2021-10-13T17:27:58.011101Z","shell.execute_reply.started":"2021-10-13T17:27:57.689734Z","shell.execute_reply":"2021-10-13T17:27:58.009945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Observe the shape of the training and validation Datasets.  This is the batch size (128) and number of pixel columns (784).","metadata":{}},{"cell_type":"code","source":"print(train_ds.element_spec)\nprint(val_ds.element_spec)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:58.012438Z","iopub.execute_input":"2021-10-13T17:27:58.012822Z","iopub.status.idle":"2021-10-13T17:27:58.023123Z","shell.execute_reply.started":"2021-10-13T17:27:58.012786Z","shell.execute_reply":"2021-10-13T17:27:58.022187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Build a sequential CNN with preprocessing layers\n\nThe reshaping and rescaling tasks are included in the model.  These preprocessing layers can then be saved and exported with the CNN.  This is the preferred approach only if you're training on a GPU. \n\nA CNN requires image dimensions and a color channel to function.  Here inputs are reshaped to include the grayscale color channel.  \n\nInputs need to be normalized to a range of 0 to 1.  Divide the pixel values by 255 to achieve this scale.  Insert this layer after reshaping. \n\nIn this example two convolutional layers are used, each with batch normalization, max pooling, and a dropout layer. A final classification layer is added with units the size of the number of classes.","metadata":{}},{"cell_type":"code","source":"# Reshape to (height, width, color channel)\nreshape_layer = layers.Reshape((28, 28, 1))\n# Include the decimal point when dividing to output floats \nrescale_layer = layers.experimental.preprocessing.Rescaling(scale=1./255)\n\nmodel = Sequential([\n                    layers.InputLayer(input_shape=[28, 28]),\n                    reshape_layer,\n                    rescale_layer,\n                    layers.Conv2D(128, kernel_size=3, activation='relu'),                                                 \n                    layers.BatchNormalization(), # Normalizes layer activations                     \n                    layers.MaxPooling2D(2,2), # Downsamples the feature map                    \n                    layers.Dropout(0.5), # Reduces overfitting\n                    layers.Conv2D(128, kernel_size=3, activation='relu'),\n                    layers.BatchNormalization(),\n                    layers.MaxPooling2D(2,2),\n                    layers.Dropout(0.5), \n                    layers.Flatten(),  \n                    layers.Dense(NUM_LABELS, activation='softmax')\n])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:27:58.026797Z","iopub.execute_input":"2021-10-13T17:27:58.027123Z","iopub.status.idle":"2021-10-13T17:27:58.161982Z","shell.execute_reply.started":"2021-10-13T17:27:58.027097Z","shell.execute_reply":"2021-10-13T17:27:58.161163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Train the CNN model \nSparse categorical crossentropy is used as the loss function with Adam as the optimizer.","metadata":{}},{"cell_type":"code","source":"model.compile(\n    loss='sparse_categorical_crossentropy', \n    optimizer=Adam(lr=1e-3),\n    metrics=['accuracy']\n)\n\nEPOCHS = 20\n\nhistory = model.fit(\n    train_ds,\n    epochs=EPOCHS, \n    validation_data=val_ds\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:29:49.021341Z","iopub.execute_input":"2021-10-13T17:29:49.021677Z","iopub.status.idle":"2021-10-13T17:31:58.658076Z","shell.execute_reply.started":"2021-10-13T17:29:49.021645Z","shell.execute_reply":"2021-10-13T17:31:58.657188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Visualize and evaluate the training and validation results","metadata":{}},{"cell_type":"code","source":"# Assign the loss and accuracy metrics\ntrain_loss = history.history['loss']\nval_loss = history.history['val_loss']\ntrain_acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:31:58.65975Z","iopub.execute_input":"2021-10-13T17:31:58.660014Z","iopub.status.idle":"2021-10-13T17:31:58.666524Z","shell.execute_reply.started":"2021-10-13T17:31:58.659989Z","shell.execute_reply":"2021-10-13T17:31:58.664704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a function to plot the metrics\ndef plot_history(train_history, val_history, label):\n    plt.plot(train_history, label=f'Training {label}')\n    plt.plot(val_history, label=f'Validation {label}')\n    plt.xlabel('Epochs')\n    plt.legend()\n    return plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:31:58.66885Z","iopub.execute_input":"2021-10-13T17:31:58.669343Z","iopub.status.idle":"2021-10-13T17:31:58.676315Z","shell.execute_reply.started":"2021-10-13T17:31:58.669302Z","shell.execute_reply":"2021-10-13T17:31:58.675122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The training and validation plots should converge for optimal results.","metadata":{}},{"cell_type":"code","source":"plot_history(train_loss, val_loss, 'Loss')\nplot_history(train_acc, val_acc, 'Accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:31:58.678009Z","iopub.execute_input":"2021-10-13T17:31:58.678375Z","iopub.status.idle":"2021-10-13T17:31:58.994282Z","shell.execute_reply.started":"2021-10-13T17:31:58.678341Z","shell.execute_reply":"2021-10-13T17:31:58.993482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Create a Dataset from the test data","metadata":{}},{"cell_type":"markdown","source":"### 6.1 Load the test .csv file ","metadata":{}},{"cell_type":"code","source":"path_test = '../input/digit-recognizer/test.csv'\ndf_test = pd.read_csv(path_test)\nprint(df_test.shape)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:31:58.995662Z","iopub.execute_input":"2021-10-13T17:31:58.99605Z","iopub.status.idle":"2021-10-13T17:32:01.345693Z","shell.execute_reply.started":"2021-10-13T17:31:58.99601Z","shell.execute_reply":"2021-10-13T17:32:01.3449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = tf.data.Dataset.from_tensor_slices((df_test))\ntest_ds = configure_dataset(test_ds, test=True) # Set test argument to True so that remainder samples are not dropped\ntest_ds.element_spec","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:32:01.346952Z","iopub.execute_input":"2021-10-13T17:32:01.3473Z","iopub.status.idle":"2021-10-13T17:32:01.581581Z","shell.execute_reply.started":"2021-10-13T17:32:01.347266Z","shell.execute_reply":"2021-10-13T17:32:01.580583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Predict label outcomes and submit results","metadata":{}},{"cell_type":"code","source":"y_pred = np.argmax(model.predict(test_ds), axis=-1) # Returns the highest probability label for each image\ny_pred = pd.DataFrame(y_pred)\ny_pred.columns = ['Label']\nprint(y_pred.shape)\ny_pred.head()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:32:01.583044Z","iopub.execute_input":"2021-10-13T17:32:01.583434Z","iopub.status.idle":"2021-10-13T17:32:03.133147Z","shell.execute_reply.started":"2021-10-13T17:32:01.583394Z","shell.execute_reply":"2021-10-13T17:32:03.132327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = pd.DataFrame(y_pred.index + 1)\nimage_id.columns = ['ImageId']\nimage_id.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:32:03.135116Z","iopub.execute_input":"2021-10-13T17:32:03.135459Z","iopub.status.idle":"2021-10-13T17:32:03.145717Z","shell.execute_reply.started":"2021-10-13T17:32:03.135423Z","shell.execute_reply":"2021-10-13T17:32:03.144698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([image_id, y_pred], axis=1)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T17:32:03.14729Z","iopub.execute_input":"2021-10-13T17:32:03.147713Z","iopub.status.idle":"2021-10-13T17:32:03.204641Z","shell.execute_reply.started":"2021-10-13T17:32:03.147676Z","shell.execute_reply":"2021-10-13T17:32:03.203688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}