{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notes:   \n\n##### This was an experiment to try to better prepare for a project I will be starting soon at my workplace so if you have any thoughts about my work please share via a comment or a vote! A few keys that allow this model to preform highly are conducting proper experiments to determine model archetecture (not seen here for computations sake but very similar to Chris' work seen here https://www.kaggle.com/code/cdeotte/how-to-choose-cnn-architecture-mnist), augmenting images slightly to create more images, and lastly an understanding of the limit of your model (more about this at the last note)\n\n##### This model CAN be slightly improved however! The way you can do it is by ensembling this model with various other ones, you can find an example of how to ensemble CNNs here https://www.kaggle.com/code/cdeotte/25-million-images-0-99757-mnist in another one of Chris' works.\n\n##### How good can a model be on MNIST? This is a very important concept and many are tricked by \"perfect models\" in the code section. A model cannot be perfect without using the full MNIST data as your submission. Realistically the best you can get a model is somewhere around 99.80%. The reasoning for this is very simple the mislabeling of certain images within the data will cause aa little confusion in the modeling process and there is really no such thing as perfect in practice. This is why we have the saying \"If it sounds too good to be true, it probably is.\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Import data and packages 📚","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, Lambda, MaxPooling2D, MaxPool2D # convolution layers\nfrom keras.layers import Dense, Dropout, Flatten # core layers\nfrom tensorflow.keras.layers import BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils.np_utils import to_categorical\nfrom keras.utils.vis_utils import plot_model\nfrom IPython.display import Image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nsub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\nprint(\"Data is ready!\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training data size is {train.shape}\\nTesting data size is {test.shape}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing data for modeling","metadata":{}},{"cell_type":"code","source":"X = train.drop(['label'], 1).values\ny = train['label'].values\n\ntest_x = test.values","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X / 255.0\ntest_x = test_x / 255.0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\n# canal = 1 => For gray scale\nX = X.reshape(-1,28,28,1)\ntest_x = test_x.reshape(-1,28,28,1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\ny = to_categorical(y)\n\nprint(f\"Label size {y.shape}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_val.shape, y_train.shape, y_val.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train__ = X_train.reshape(X_train.shape[0], 28, 28)\n\nfig, axis = plt.subplots(1, 4, figsize=(20, 10))\nfor i, ax in enumerate(axis.flat):\n    ax.imshow(X_train__[i], cmap='binary')\n    digit = y_train[i].argmax()\n    ax.set(title = f\"Real Number is {digit}\");","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = np.mean(X_train)\nstd = np.std(X_train)\n\ndef standardize(x):\n    return (x-mean)/std","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling\n\n##### You can learn more about how to come up with this model archetecture here https://www.kaggle.com/code/cdeotte/how-to-choose-cnn-architecture-mnist","metadata":{}},{"cell_type":"code","source":"epochs = 50\nbatch_size = 128","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Sequential()\n\nmodel.add(Conv2D(filters=64, kernel_size=(5, 5), padding='Same', activation='relu', input_shape=(28, 28, 1)))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters=64, kernel_size=(5, 5), padding='Same', activation='relu'))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same',  activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Dense(10, activation ='softmax'))\n\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With data augmentation to prevent overfitting\n\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\n#datagen.fit(X_train)\ntrain_gen = datagen.flow(X_train, y_train, batch_size=batch_size)\nval_gen = datagen.flow(X_val, y_val, batch_size=batch_size)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the model\nhistory = model.fit_generator(train_gen, \n                              epochs = epochs, \n                              steps_per_epoch = X_train.shape[0] // batch_size,\n                              validation_data = val_gen,\n                              validation_steps = X_val.shape[0] // batch_size)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summarize Results","metadata":{}},{"cell_type":"code","source":"plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)\nImage(\"model_plot.png\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(2,1, figsize=(18, 10))\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10)) # Set Figure\n\ny_pred = model.predict(X_val) # Predict encoded label as 2 => [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]\n\nY_pred = np.argmax(y_pred, 1) # Decode Predicted labels\nY_val = np.argmax(y_val, 1) # Decode labels\n\nmat = confusion_matrix(Y_val, Y_pred) # Confusion matrix\n\n# Plot Confusion matrix\nsns.heatmap(mat.T, square=True, annot=True, cbar=False, cmap=plt.cm.Blues)\nplt.xlabel('Predicted Values')\nplt.ylabel('True Values');\nplt.show();","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_val)\nX_val__ = X_val.reshape(X_val.shape[0], 28, 28)\n\nfig, axis = plt.subplots(4, 4, figsize=(12, 14))\nfor i, ax in enumerate(axis.flat):\n    ax.imshow(X_val__[i], cmap='binary')\n    ax.set(title = f\"Real Number is {y_val[i].argmax()}\\nPredict Number is {y_pred[i].argmax()}\");","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and submission","metadata":{}},{"cell_type":"code","source":"pred = model.predict(test_x, verbose=1)\nclasses = np.argmax(pred, axis=1)\n\ndisplay(classes)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(classes))\nsub['Label'] = classes\nsub.to_csv(\"submission.csv\", index=False)\nsub.head()","metadata":{},"execution_count":null,"outputs":[]}]}