{"cells":[{"metadata":{},"cell_type":"markdown","source":"Solving MNIST\nProblem statement: Predict digit value given handwriting picture (28 x 28 pixel) Type: Multiclass Classification (0-9 digit) Performance metric: Accuracy;\n\nI will approach this problem using Convolutional Neural Network (CNN). I want to beat my previous CNN architecture that has score in public leaderboard 0.98371. My previous CNN could be found [here](https://www.kaggle.com/ajisamudra/solving-mnist-using-cnn-mlp-and-stacking).\n\nWhat I did in this notebook:\nData pre-processing: Normalize the data to range 0-1 so it could be faster to converge.\nCompare several CNN architecture to beat my previous CNN."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Library\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.models import Sequential\nimport numpy as np\nimport pandas as pd\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport datetime\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split \n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read file\ndf_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\nX_train = df_train.iloc[:,1:]\ny_train = df_train.iloc[:,0]\n\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train,\n                                                    stratify = y_train,\n                                                    test_size=0.2, random_state = 41)\n\nX_train.shape, y_train.shape, X_val.shape, y_val.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Prepare dataset for CNN\n\n# Get data from previous model\narr_X_train = X_train.to_numpy()\narr_X_val = X_val.to_numpy()\narr_y_train = y_train.to_numpy()\narr_y_val = y_val.to_numpy()\n\n# Get data from csv file and normalize it\narr_X_test = df_test.to_numpy()\n\n# Normalize data\narr_X_train = arr_X_train / 255\narr_X_val = arr_X_val / 255\narr_X_test = arr_X_test / 255\n\n# input image dimensions\nimg_rows, img_cols = 28, 28\nnum_classes = 10\n\n# Reshape the array to (28,28,1)\nX_train = arr_X_train.reshape(arr_X_train.shape[0], img_rows, img_cols, 1)\nX_val = arr_X_val.reshape(arr_X_val.shape[0], img_rows, img_cols, 1)\nX_test = arr_X_test.reshape(arr_X_test.shape[0], img_rows, img_cols, 1)\ninput_shape = (img_rows, img_cols, 1)\n\n# convert class vectors to binary class matrices\ny_train = keras.utils.to_categorical(y_train, num_classes)\ny_val = keras.utils.to_categorical(y_val, num_classes)\n\nX_train.shape, y_train.shape, X_val.shape, y_val.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define CNN architecture\n\n# This is my previous CNN architecture\n# The model is constructed by several layers\n# I use relu activation and he initializer to get faster converge time\n# The first layer is convolutional layer with 16 filter, 3x3 kernel size, 1 stride\n# The second layer is the same with the first layer. They are supposed to capture feature maps from digit image.\n# The third layer is pooling with maximum aggregation. It is used to reduce the size of feature maps.\n# The fourth layer is convolutional layer for the result from maximum pooling.\n# The next layer is dense neural network with 256 units, before feeding this layer with data I need to flatten first the data\n# The last layer / output layer is dense neural network with 10 units (the same with number of class)\n\ncnn_model = keras.models.Sequential([\n    keras.layers.Conv2D(16, kernel_size=(3, 3), activation='relu', input_shape=input_shape, kernel_initializer= 'he_normal'),\n    keras.layers.Conv2D(16, kernel_size=(3, 3), activation='relu', kernel_initializer= 'he_normal'),\n    keras.layers.MaxPooling2D(pool_size=(3, 3)),\n    keras.layers.Conv2D(16, kernel_size=(3, 3), activation='relu', kernel_initializer= 'he_normal'),\n    keras.layers.Flatten(),\n    keras.layers.Dense(256, activation='relu', kernel_initializer= 'he_normal'),\n    keras.layers.Dense(num_classes, activation='softmax')\n])\n\n# Compile the model\ncnn_model.compile(optimizer= 'adam',\n                  loss=keras.losses.categorical_crossentropy,\n                  metrics= ['accuracy']\n                 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\ncnn_model.fit(X_train, y_train, epochs=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Evaluate model in validation dataset\ntest_loss, test_accuracy = cnn_model.evaluate(X_val, y_val, verbose= 0)\nprint(\"CNN Accuracy on Validation: {}\".format(test_accuracy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to train and evaluate\ndef train_and_validate(model, epochs):\n    model.fit(X_train, y_train, epochs=epochs)\n    test_loss, test_accuracy = model.evaluate(X_val, y_val, verbose= 0)\n    print(\"CNN Accuracy on Validation: {}\".format(test_accuracy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Challenger architecture\n\ncnn2_model = keras.models.Sequential([\n    keras.layers.Conv2D(64, 7, input_shape=input_shape, activation='relu',  padding='same', kernel_initializer= 'he_normal'),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Conv2D(128, 3, activation='relu',  padding='same', kernel_initializer= 'he_normal'),\n    keras.layers.Conv2D(128, 3, activation='relu',  padding='same', kernel_initializer= 'he_normal'),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Conv2D(256, 3, activation='relu',  padding='same', kernel_initializer= 'he_normal'),\n    keras.layers.Conv2D(256, 3, activation='relu',  padding='same', kernel_initializer= 'he_normal'),\n    keras.layers.MaxPooling2D(2),    \n    keras.layers.Flatten(),\n    keras.layers.Dense(128, activation='relu', kernel_initializer= 'he_normal'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(64, activation='relu', kernel_initializer= 'he_normal'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(num_classes, activation='softmax')\n])\n\n# Compile the model\ncnn2_model.compile(optimizer= 'adam',\n                  loss=keras.losses.categorical_crossentropy,\n                  metrics= ['accuracy']\n                 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train and evaluate \ntrain_and_validate(cnn2_model, 12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction for submission\narr_y_pred = cnn2_model.predict(X_test)\n\n# Create list of prediction\ny_pred = []\nfor i in range(len(arr_y_pred)):\n    y_pred.append(np.argmax(arr_y_pred[i]))\n\n# Create file submisssion from CNN\nsubmission_cnn = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\nsubmission_cnn.iloc[:,1] = (y_pred)\nsubmission_cnn.to_csv(\"submission_cnn2.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}