{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom keras import layers\nfrom keras import models\nfrom keras.utils import to_categorical\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Contents:\n\n1. Import dataset\n2. Basic Random forest implementation \n3. Simple Artificial Neural Network implementation\n4. Convolutional Neural Network implementation"},{"metadata":{},"cell_type":"markdown","source":"---\n\n## 1. Import dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/digit-recognizer/\"\ntrain_df = pd.read_csv(PATH + 'train.csv')\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train_df.iloc[:, 1:].values\ny = train_df.iloc[:, 0].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# view an example image from the dataset\nplt.imshow(X[1600].reshape((28, 28)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n\n## 1. Basic random forest model"},{"metadata":{},"cell_type":"markdown","source":"#### Fit a default random forest to our data and make predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# split training data into training and validation splits\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=12) \n\n# form a basic random forest (default hyper-parameters)\nrf_model = RandomForestClassifier()\n\n# fit model with training data\nrf_model.fit(X_train, y_train)\n\n# predict output labels\npreds = rf_model.predict(X_val)\n\n# view accuracy\naccuracy_score(preds, y_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"96.3% is not bad for almost no effort... Let's submit this on the test set as an initial baseline.\n\n---"},{"metadata":{},"cell_type":"markdown","source":"#### Import test data and make predictions using a model trained on the entire training set"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/digit-recognizer/\"\ntest_df = pd.read_csv(PATH + 'test.csv')\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Re-train the random forrest model, but with the whole test set this time."},{"metadata":{"trusted":true},"cell_type":"code","source":"rf_model = RandomForestClassifier()\nrf_model.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/digit-recognizer/\"\ntest_df = pd.read_csv(PATH + 'test.csv')\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = test_df.values\ntest_preds = rf_model.predict(X_test)\n\ntest_preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Format predictions as submission file, and output submission file for uploading."},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = np.arange(1, len(test_preds) + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'Imageid' : image_id, 'Label' : test_preds})\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('random_forrest_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## 2. Simple Deep Neural Network\n\nWhen creating a neural network model, we have to process our data slightly differently. For our labels, we need to one-hot encode the categorical variables. This gives us ten sparse vectors, rather than one dense column of ten categorical values for the given output."},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split data into training / validation splits\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=12) \n\n# standardise / normalise each of our pixels\nX_train = X_train.astype('float32') / 255.0\nX_val = X_val.astype('float32') / 255.0\n\n# one-hot encode our labels\ny_train_one_hot = to_categorical(y_train)\ny_val_one_hot = to_categorical(y_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For our model, we need an input shape equal to the number of columns within our training data, which is 28 * 28 = 784. An easier way of putting this into our model is using X_train.shape[1] instead.\n\nIt should also be noted that since we have multiple categorical output labels, we must also use a softmax output layer too. Rather than a sigmoid single value output, a softmax activation outputs a probability distributions over the ten output classes, with the maximum value corresponding to the predicted output class. The values of all ten outputs for each sample prediction will add to 1.0."},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))\nmodel.add(layers.Dense(32, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(X_train, y_train_one_hot, epochs=40, \n                    batch_size=512, validation_data=(X_val, y_val_one_hot))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_dict = history.history\n\ntrg_loss = hist_dict['loss']\nval_loss = hist_dict['val_loss']\n\ntrg_acc = hist_dict['accuracy']\nval_acc = hist_dict['val_accuracy']\n\nepochs = range(1, len(trg_acc) + 1)\n\nplt.plot(epochs, trg_loss, marker='o', label='Training Loss')\nplt.plot(epochs, val_loss, marker='x', label='Validation Loss')\nplt.title(\"Training / Validation Loss\")\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Epochs\")\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(epochs, trg_acc, marker='o', label='Training Accuracy')\nplt.plot(epochs, val_acc, marker='^', label='Validation Accuracy')\nplt.title(\"Training / Validation Loss\")\nplt.ylabel(\"Accuracy\")\nplt.xlabel(\"Epochs\")\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train on the full training set, and make predictions on unseen test set"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_full = X.astype('float32') / 255.0\ny_one_hot = to_categorical(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Dense(64, activation='relu', input_shape=(X_train.shape[1],)))\nmodel.add(layers.Dense(32, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\n\nmodel.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model.fit(X_full, y_one_hot, epochs=40, batch_size=512)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_dict = history.history\n\ntrg_loss = history.history['loss']\ntrg_acc = history.history['accuracy']\n\nepochs = range(1, len(trg_acc) + 1)\n\nfig, ax = plt.subplots(1, 2, figsize=(12, 4))\nax[0].plot(epochs, trg_loss, label='Training Loss')\nax[1].plot(epochs, trg_acc, label='Training Accuracy')\nax[0].set_ylabel('Loss')\nax[1].set_ylabel('Accuracy')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/digit-recognizer/\"\ntest_df = pd.read_csv(PATH + 'test.csv')\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make our predictions and convert back into sparse from one-hot encoded\nX_test = test_df.values\ntest_preds = model.predict(X_test)\ntest_preds = np.argmax(test_preds, axis=1)\n\ntest_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = np.arange(1, len(test_preds) + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'Imageid' : image_id, 'Label' : test_preds})\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('deep_neural_network.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Convolutional Neural Network Implementation"},{"metadata":{},"cell_type":"markdown","source":"For a convolutional neural network, we need to pass the data into our model in the form (n_samples, img_height, img_width, img_dimensions). In order to do this, we need to reshape the original training data, X. Just like the previous ANN implementation, we'll also standardise our image pixel values by dividing each by 255."},{"metadata":{"trusted":true},"cell_type":"code","source":"# reshape and standardise our data\nX = X.reshape((-1, 28, 28, 1))\nX_std = X.astype('float32') / 255.0\n\n# obtain test set for predictions later\nX_test = test_df.values\nX_test = X_test.reshape((-1, 28, 28, 1))\nX_test_std = X_test.astype('float32') / 255.0\n\n# one-hot encode our labels\ny_one_hot = to_categorical(y)\n\nprint(f\"Training data: {X.shape}, Training Labels: {y_one_hot.shape} \\n\"\n      f\"Test shape: {X_test.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split data into training / validation splits\nX_train, X_val, y_train, y_val = train_test_split(X_std, y_one_hot, test_size=0.2, random_state=12) \n\nprint(f\"X_train: {X_train.shape}, y_train: {y_train.shape} \\n\"\n      f\"X_val: {X_val.shape}, y_val: {y_val.shape} \\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We'll form our convolutional neural network using three Conv layers, the first two of which are followed by a Max Pooling layer. Why do we need max pooling layers? We do this to reduce the overall number of co-efficients that are model needs to process during training, otherwise we end up with an extremely large and inefficient model. This would take very long to train, whilst giving a poor return in performance.\n\nBy downsampling (applying max pooling layers), we are also inducing spatial-filter hierarchies throughout our model, which means our filters are being applied across a larger proportion of the original image towards the later layers. This subsequently helps us better generalise images and form a better overall understanding of what we're feeding into it. We could have performed this another way - for instance, use of larger strides within our model layers, or conversely we could have applied average pooling (as opposed to max pooling)."},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_model = models.Sequential()\ncnn_model.add(layers.Conv2D(32, (3, 3), activation='relu', \n                            input_shape=(X.shape[1], X.shape[2], X.shape[3])))\ncnn_model.add(layers.MaxPooling2D((2, 2)))\ncnn_model.add(layers.Conv2D(64, (3, 3), activation='relu'))\ncnn_model.add(layers.MaxPooling2D((2, 2)))\ncnn_model.add(layers.Conv2D(64, (3, 3), activation='relu'))\ncnn_model.add(layers.Flatten())\ncnn_model.add(layers.Dense(64, activation='relu'))\ncnn_model.add(layers.Dense(10, activation='softmax'))\n\ncnn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = cnn_model.fit(X_train, y_train, epochs=20, \n                        batch_size=128, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_dict = history.history\n\ntrg_loss = hist_dict['loss']\nval_loss = hist_dict['val_loss']\n\ntrg_acc = hist_dict['accuracy']\nval_acc = hist_dict['val_accuracy']\n\nepochs = range(1, len(trg_acc) + 1)\n\n# plot losses and accuracies for training and validation \nfig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(1, 2, 1)\nplt.plot(epochs, trg_loss, marker='o', label='Training Loss')\nplt.plot(epochs, val_loss, marker='x', label='Validation Loss')\nplt.title(\"Training / Validation Loss\")\nax.set_ylabel(\"Loss\")\nax.set_xlabel(\"Epochs\")\nplt.legend(loc='best')\n\nax = fig.add_subplot(1, 2, 2)\nplt.plot(epochs, trg_acc, marker='o', label='Training Accuracy')\nplt.plot(epochs, val_acc, marker='^', label='Validation Accuracy')\nplt.title(\"Training / Validation Accuracy\")\nax.set_ylabel(\"Accuracy\")\nax.set_xlabel(\"Epochs\")\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Adjust various parameters on our model - change depth of convolutional layers."},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_model = models.Sequential()\ncnn_model.add(layers.Conv2D(32, (3, 3), activation='relu', \n                            input_shape=(X.shape[1], X.shape[2], X.shape[3])))\ncnn_model.add(layers.MaxPooling2D((2, 2)))\ncnn_model.add(layers.Conv2D(64, (3, 3), activation='relu'))\ncnn_model.add(layers.MaxPooling2D((2, 2)))\ncnn_model.add(layers.Conv2D(128, (3, 3), activation='relu'))\ncnn_model.add(layers.Flatten())\ncnn_model.add(layers.Dense(128, activation='relu'))\ncnn_model.add(layers.Dense(10, activation='softmax'))\n\ncnn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = cnn_model.fit(X_train, y_train, epochs=20, \n                        batch_size=128, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_dict = history.history\n\ntrg_loss = hist_dict['loss']\nval_loss = hist_dict['val_loss']\n\ntrg_acc = hist_dict['accuracy']\nval_acc = hist_dict['val_accuracy']\n\nepochs = range(1, len(trg_acc) + 1)\n\n# plot losses and accuracies for training and validation \nfig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(1, 2, 1)\nplt.plot(epochs, trg_loss, marker='o', label='Training Loss')\nplt.plot(epochs, val_loss, marker='x', label='Validation Loss')\nplt.title(\"Training / Validation Loss\")\nax.set_ylabel(\"Loss\")\nax.set_xlabel(\"Epochs\")\nplt.legend(loc='best')\n\nax = fig.add_subplot(1, 2, 2)\nplt.plot(epochs, trg_acc, marker='o', label='Training Accuracy')\nplt.plot(epochs, val_acc, marker='^', label='Validation Accuracy')\nplt.title(\"Training / Validation Accuracy\")\nax.set_ylabel(\"Accuracy\")\nax.set_xlabel(\"Epochs\")\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Final validation accuracy: {val_acc[-1]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Finally, lets train our CNN on the entire dataset and make a submission for the test set."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_cnn_model():\n    model = models.Sequential()\n    model.add(layers.Conv2D(32, (3, 3), activation='relu', \n                            input_shape=(X.shape[1], X.shape[2], X.shape[3])))\n    model.add(layers.MaxPooling2D((2, 2)))\n    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n    model.add(layers.MaxPooling2D((2, 2)))\n    model.add(layers.Conv2D(128, (3, 3), activation='relu'))\n    model.add(layers.Flatten())\n    model.add(layers.Dense(128, activation='relu'))\n    model.add(layers.Dense(10, activation='softmax'))\n    model.compile(optimizer='rmsprop', \n                  loss='categorical_crossentropy', \n                  metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_full = X.astype('float32') / 255.0\ny_full = to_categorical(y)\n\ncnn_model = create_cnn_model()\ncnn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = cnn_model.fit(X_full, y_full, epochs=20, batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_dict = history.history\n\ntrg_loss = history.history['loss']\ntrg_acc = history.history['accuracy']\n\nepochs = range(1, len(trg_acc) + 1)\n\nfig, ax = plt.subplots(1, 2, figsize=(12, 4))\nax[0].plot(epochs, trg_loss, label='Training Loss')\nax[1].plot(epochs, trg_acc, label='Training Accuracy')\nax[0].set_ylabel('Loss')\nax[1].set_ylabel('Accuracy')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Import test dataset and predict output labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/digit-recognizer/\"\ntest_df = pd.read_csv(PATH + 'test.csv')\ntest_df.shape\n\n# format our test data appropriately for our CNN model\nX_test = test_df.values\nX_test = X_test.reshape((-1, 28, 28, 1))\nX_test_std = X_test.astype('float32') / 255.0\n\n\ntest_preds = cnn_model.predict(X_test_std)\ntest_preds = np.argmax(test_preds, axis=1)\n\ntest_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'Imageid' : np.arange(1, len(test_preds) + 1), \n                           'Label' : test_preds})\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('convolutional_neural_network.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}