{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nThe aim of this notebook is to create an image classification TensorFlow 2 neural network from first principles. Accuracy is not the highest priority in this notebook. Achieving a solid understanding in CNN's with TensorFlow 2 is the main priority.   "},{"metadata":{},"cell_type":"markdown","source":"# References\n\n* https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\n* https://www.tensorflow.org/tutorials/keras/classification\n* https://www.tensorflow.org/tensorboard/tensorboard_in_notebooks\n* https://www.kaggle.com/shivam1600/tensorboard-on-kaggle"},{"metadata":{},"cell_type":"markdown","source":"# Import Packages and Data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#Import all the required libraries\nimport os\nimport datetime\nimport multiprocessing\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\n\nfrom sklearn.model_selection import train_test_split\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nsns.set(style='darkgrid', context='notebook', palette='deep')\nprint(\"TensorFlow version: {}\".format(tf.__version__))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create dataframes from all the CSV data\ntrain_df = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"../input/digit-recognizer/test.csv\")\nsample_df = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Data for Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separate into training data into images and labels\ny_train = train_df[\"label\"]\nx_train = train_df.drop(labels = [\"label\"], axis=1)\n\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set the random seed\nrandom_seed = 2\n# Split train data into training and validation\nx_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.1, random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert pixel intensities from range 0-255 to 0-1 for the model\nx_train = x_train / 255\nx_val = x_val / 255\ntest_df = test_df / 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert single line image data into 2D arrays\nx_train = np.reshape(x_train.values, (-1, 28, 28, 1))\nprint(\"Training images shape: {}\".format(np.shape(x_train)))\nx_val = np.reshape(x_val.values, (-1, 28, 28, 1))\nprint(\"Validation images shape: {}\".format(np.shape(x_val)))\ntest_df = np.reshape(test_df.values, (-1, 28, 28, 1))\nprint(\"Test images shape: {}\".format(np.shape(test_df)))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# View the Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the training set images\nplt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(x_train[i], cmap=plt.cm.binary)\n    plt.xlabel(str(y_train.iloc[i]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create the Convolutional Neural Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the models layer structure\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Conv2D(filters=16, kernel_size=(5,5), padding=\"same\", activation=\"relu\", input_shape=(28,28,1)),\n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Conv2D(filters=32, kernel_size=(5,5), padding=\"same\", activation=\"relu\"),\n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), padding=\"same\", activation=\"relu\"),\n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(128, activation=\"relu\"),\n    tf.keras.layers.Dense(10)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer=\"adam\",\n             loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n             metrics=[\"accuracy\"])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup TensorBoard"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clear any logs from previous runs\n!rm -rf ./logs/ \n!mkdir ./logs/\n\n# Download Ngrok to tunnel the tensorboard port to an external port\n!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n!unzip ngrok-stable-linux-amd64.zip \n\n# Run tensorboard as well as Ngrox (for tunneling as non-blocking processes)\npool = multiprocessing.Pool(processes = 10)\nresults_of_processes = [pool.apply_async(os.system, args=(cmd, ), callback = None )\n                        for cmd in [\n                        f\"tensorboard --logdir ./logs/ --host 0.0.0.0 --port 6006 &\",\n                        \"./ngrok http 6006 &\"\n                        ]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Click on the link output below to view the training in Tensorboard\n! curl -s http://localhost:4040/api/tunnels | python3 -c \\\n    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tensorboard callback for training graphs\nlogdir = os.path.join(\"logs/fit/\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\ntensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The following method of viewing Tensorboard is currently not working in kaggle notebooks, so it's commented out\n\n# # Display training results in Tensorboard\n# %load_ext tensorboard\n# %tensorboard --logdir logs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the Model\nNote: While the models training you can view the results in Tensorboard using the link above "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\nmodel.fit(x=x_train, y=y_train, epochs=5, validation_data=(x_val, y_val), callbacks=[tensorboard_callback])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make predictions on the validation set with the trained model\nprobability_model = tf.keras.Sequential([model, tf.keras.layers.Softmax()])\npredictions = probability_model.predict(x_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Display Some Evaluation Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Functions for displaying the prediction results\nclass_names = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\ndef plot_image(i, predictions_array, y_val, img):\n  true_label, img = y_val.iloc[i], img[i]\n  plt.grid(False)\n  plt.xticks([])\n  plt.yticks([])\n\n  plt.imshow(img, cmap=plt.cm.binary)\n\n  predicted_label = np.argmax(predictions_array)\n  if predicted_label == true_label:\n    color = 'blue'\n  else:\n    color = 'red'\n\n  plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n                                100*np.max(predictions_array),\n                                class_names[true_label]),\n                                color=color)\n\ndef plot_value_array(i, predictions_array, y_val):\n  true_label = y_val.iloc[i]\n  plt.grid(False)\n  plt.xticks(range(10))\n  plt.yticks([])\n  thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n  plt.ylim([0, 1])\n  predicted_label = np.argmax(predictions_array)\n\n  thisplot[predicted_label].set_color('red')\n  thisplot[true_label].set_color('blue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the first X test images, their predicted labels, and the true labels.\n# Color correct predictions in blue and incorrect predictions in red.\nnum_rows = 8\nnum_cols = 3\nnum_images = num_rows*num_cols\nplt.figure(figsize=(2*2*num_cols, 2*num_rows))\nfor i in range(num_images):\n  plt.subplot(num_rows, 2*num_cols, 2*i+1)\n  plot_image(i, predictions[i], y_val, x_val)\n  plt.subplot(num_rows, 2*num_cols, 2*i+2)\n  plot_value_array(i, predictions[i], y_val)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot the Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(detected_ids, y_df):\n\tconf_matrix = tf.math.confusion_matrix(labels=y_df, predictions=detected_ids)\n\tfig = plt.figure(figsize=(10, 8))\n\tsns.heatmap(conf_matrix, annot=True)\n\tplt.title(\"Confusion Matrix\")\n\tplt.ylabel('True Score')\n\tplt.xlabel('Predicted Score')\n\treturn fig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract the final prediction by getting the index with maximum probability\npred = np.apply_along_axis(np.argmax, 1, predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(pred, y_val)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Process the Test Data and Submit the Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process the test images through the model\ntest_pred = model.predict(test_df)\n# Select the predictions with highest probability\ntest_pred = np.argmax(test_pred, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create an array of predicitons and their respective image IDs\noutput_array = np.array([[imageId, label] for (imageId, label) in zip(list(range(1,28001)), test_pred)])\n# Convert the output array into a datafram\noutput_df = pd.DataFrame(output_array, columns=[\"ImageId\", \"label\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save the output to a CSV\noutput_df.to_csv(\"./mnist_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}