{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\n# Sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix\n\n# Tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.utils.np_utils import to_categorical\n\n# estipulate the seeds\nseed = 100\nnp.random.seed(seed)\ntf.random.set_seed(seed)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-04T14:48:11.772445Z","iopub.execute_input":"2021-12-04T14:48:11.773090Z","iopub.status.idle":"2021-12-04T14:48:11.784277Z","shell.execute_reply.started":"2021-12-04T14:48:11.773050Z","shell.execute_reply":"2021-12-04T14:48:11.783217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# explore files and paths\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-04T14:41:12.935387Z","iopub.execute_input":"2021-12-04T14:41:12.935603Z","iopub.status.idle":"2021-12-04T14:41:12.942725Z","shell.execute_reply.started":"2021-12-04T14:41:12.935576Z","shell.execute_reply":"2021-12-04T14:41:12.941679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data explorarion","metadata":{}},{"cell_type":"code","source":"# read files\ndata_train = pd.read_csv('../input/digit-recognizer/train.csv')\ndata_test = pd.read_csv('../input/digit-recognizer/test.csv')\n# data can also be loaded with\n#from tensorflow.keras.datasets import mnist\n#(X_train,y_train),(X_test,y_test) = mnist.load_data()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:12.944426Z","iopub.execute_input":"2021-12-04T14:41:12.944750Z","iopub.status.idle":"2021-12-04T14:41:18.715144Z","shell.execute_reply.started":"2021-12-04T14:41:12.944705Z","shell.execute_reply":"2021-12-04T14:41:18.714232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training dataframe size:',data_train.shape)\ndata_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:18.717700Z","iopub.execute_input":"2021-12-04T14:41:18.718073Z","iopub.status.idle":"2021-12-04T14:41:18.747364Z","shell.execute_reply.started":"2021-12-04T14:41:18.718029Z","shell.execute_reply":"2021-12-04T14:41:18.746517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*label* column corresponds to the digit value of the picture\n<br>\n*pixel0* to *pixel783* columns corresponds to each image pixels (28 x 28 = 784)","metadata":{}},{"cell_type":"code","source":"# test data does not have the label column\nprint('Test dataframe size:',data_test.shape)\ndata_test.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:18.749019Z","iopub.execute_input":"2021-12-04T14:41:18.749591Z","iopub.status.idle":"2021-12-04T14:41:18.769274Z","shell.execute_reply.started":"2021-12-04T14:41:18.749546Z","shell.execute_reply":"2021-12-04T14:41:18.768432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Any missing values?\nprint('Null values in the training set:',data_train.isnull().sum().sum())\nprint('\\nNull values in the test set:',data_test.isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:18.771347Z","iopub.execute_input":"2021-12-04T14:41:18.771996Z","iopub.status.idle":"2021-12-04T14:41:18.852732Z","shell.execute_reply.started":"2021-12-04T14:41:18.771946Z","shell.execute_reply":"2021-12-04T14:41:18.851918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label distributions\nplt.figure(figsize=(14, 6,))\nsns.countplot(x='label', data=data_train);\nplt.title('Distribution of labels in the training set');","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:18.854341Z","iopub.execute_input":"2021-12-04T14:41:18.854921Z","iopub.status.idle":"2021-12-04T14:41:19.144631Z","shell.execute_reply.started":"2021-12-04T14:41:18.854857Z","shell.execute_reply":"2021-12-04T14:41:19.143822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample pixel values:\nprint(\"Some pixels only have constant value:\", data_train[\"pixel0\"].unique())\nprint(\"\\nOther pixels contains values from 0 to 255:\\n\", data_train[\"pixel100\"].unique())","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:19.145664Z","iopub.execute_input":"2021-12-04T14:41:19.145876Z","iopub.status.idle":"2021-12-04T14:41:19.154517Z","shell.execute_reply.started":"2021-12-04T14:41:19.145849Z","shell.execute_reply":"2021-12-04T14:41:19.153679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the images\n# Figure size\nplt.figure(figsize=(6,6))\n# Subplot \nfor i in range(9):\n    img = np.asarray(data_train.iloc[i+10,1:].values.reshape((28,28))/255);\n    ax=plt.subplot(3, 3, i+1)\n    ax.grid(False)\n    plt.xticks([])\n    plt.yticks([])\n    plt.xlabel(data_train.label[i+10])\n    plt.imshow(img, cmap='gray')\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:41:19.155967Z","iopub.execute_input":"2021-12-04T14:41:19.156193Z","iopub.status.idle":"2021-12-04T14:41:19.515553Z","shell.execute_reply.started":"2021-12-04T14:41:19.156164Z","shell.execute_reply":"2021-12-04T14:41:19.514669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One-hot encoding. Transform the label into a vector (10 labels into a lenght ten vector) for computer interpretability\ntarget = to_categorical(data_train.label, num_classes=10)\n# similar as from keras.utils.np_utils import to_categorical\n#y_train=pd.get_dummies(data_train.label)\n\n# normalize pixel values to be in [0,1] and remove the label\nX=data_train.drop('label', axis=1)/255\n\n# tests data\nX_test = data_test.values/255","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:04.410020Z","iopub.execute_input":"2021-12-04T14:49:04.410640Z","iopub.status.idle":"2021-12-04T14:49:04.655140Z","shell.execute_reply.started":"2021-12-04T14:49:04.410601Z","shell.execute_reply":"2021-12-04T14:49:04.654209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Break off validation set\nX_train, X_valid, y_train, y_valid = train_test_split(X, target,\n               train_size=0.9, test_size=0.1, random_state=seed)\n\n# Reshape into image matrix 28x28 (-1 means unspecified size and the 1 for the grey image of 1 dimension, not color)\nX_train = X_train.values.reshape(-1, 28, 28, 1)\nX_valid = X_valid.values.reshape(-1, 28, 28, 1)\n\n# Save test data index for submission\ntest_index=data_test.index\n\n# Reshape the test too\nX_test = X_test.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:04.656808Z","iopub.execute_input":"2021-12-04T14:49:04.657060Z","iopub.status.idle":"2021-12-04T14:49:04.865268Z","shell.execute_reply.started":"2021-12-04T14:49:04.657031Z","shell.execute_reply":"2021-12-04T14:49:04.864449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64 # random number depend on your processor\nfrom keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n                             fill_mode='constant',\n                             rotation_range=10,  # 10 degrees\n                             zoom_range=0.1,  # 10 %\n                             width_shift_range=0.1,\n                             height_shift_range=0.1,\n                             horizontal_flip=False,\n                             vertical_flip=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:17.828647Z","iopub.execute_input":"2021-12-04T14:49:17.829422Z","iopub.status.idle":"2021-12-04T14:49:17.836732Z","shell.execute_reply.started":"2021-12-04T14:49:17.829363Z","shell.execute_reply":"2021-12-04T14:49:17.835703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#image example\naug = train_datagen.flow(X_train[1].reshape(-1, 28, 28, 1))\n\n# Plot using a for loop\nfig, axes = plt.subplots(4, 8, figsize = (12, 6))\naxes = axes.ravel()\n\nfor i in range(32):\n    aug_img = next(aug)[0]\n    axes[i].imshow(aug_img, cmap = 'gray')\n    axes[i].axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:17.838471Z","iopub.execute_input":"2021-12-04T14:49:17.838776Z","iopub.status.idle":"2021-12-04T14:49:18.982606Z","shell.execute_reply.started":"2021-12-04T14:49:17.838736Z","shell.execute_reply":"2021-12-04T14:49:18.981678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = train_datagen.flow(X_train, y_train,\n                                     batch_size=BATCH_SIZE,\n                                     shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:18.984040Z","iopub.execute_input":"2021-12-04T14:49:18.984452Z","iopub.status.idle":"2021-12-04T14:49:19.038388Z","shell.execute_reply.started":"2021-12-04T14:49:18.984417Z","shell.execute_reply":"2021-12-04T14:49:19.037401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use of data val geneartor?\nval_datagen = tf.keras.preprocessing.image.ImageDataGenerator()\nval_generator = val_datagen.flow(X_valid, y_valid,\n                                 batch_size=BATCH_SIZE,\n                                 shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:19.039971Z","iopub.execute_input":"2021-12-04T14:49:19.040225Z","iopub.status.idle":"2021-12-04T14:49:19.048769Z","shell.execute_reply.started":"2021-12-04T14:49:19.040193Z","shell.execute_reply":"2021-12-04T14:49:19.047828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define model\nmodel = tf.keras.Sequential([\n    # Convolutional layer 1\n    layers.Conv2D(filters=64, kernel_size=5, strides=1,\n                  padding='same',input_shape=[28,28,1],\n                  activation='relu'),\n    layers.MaxPool2D(pool_size=2, padding='same'),\n    layers.Dropout(rate=0.4),\n    \n    # Convolutional layer 2\n    layers.Conv2D(filters=128, kernel_size=3, strides=1,\n                  padding='same', activation='relu'),\n    layers.MaxPool2D(pool_size=4, padding='same'),\n    layers.Dropout(rate=0.4),\n    layers.Flatten(),\n    \n    # Hidden layer 3\n    layers.Dense(units=256, activation='relu'),\n    layers.Dropout(rate=0.4),\n    \n    # Output layer (softmax returns a probability distribution)\n    layers.Dense(units=10, activation='softmax')\n])\n\n# Define optimizer, loss function and accuracy metric\nmodel.compile(optimizer='adam',\n              loss='categorical_crossentropy',\n              metrics=['categorical_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:19.051041Z","iopub.execute_input":"2021-12-04T14:49:19.051638Z","iopub.status.idle":"2021-12-04T14:49:19.240592Z","shell.execute_reply.started":"2021-12-04T14:49:19.051597Z","shell.execute_reply":"2021-12-04T14:49:19.239040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Early stopping criteria\nearly_stopping = tf.keras.callbacks.EarlyStopping(\n    patience=25,\n    min_delta=0.0001,\n    restore_best_weights=True,\n)\n\n# learning rate reduction on final epochs\nlr_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc', \n                                patience=50,\n                                #verbose=1,\n                                factor=0.5,\n                                min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:19.241897Z","iopub.execute_input":"2021-12-04T14:49:19.242128Z","iopub.status.idle":"2021-12-04T14:49:19.247926Z","shell.execute_reply.started":"2021-12-04T14:49:19.242100Z","shell.execute_reply":"2021-12-04T14:49:19.246977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train model\nhistory = model.fit(train_generator,\n    validation_data=(X_valid, y_valid),\n    epochs=80,\n    batch_size = BATCH_SIZE,\n    callbacks=[early_stopping, lr_reduction],\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:49:19.249435Z","iopub.execute_input":"2021-12-04T14:49:19.249689Z","iopub.status.idle":"2021-12-04T16:34:32.630226Z","shell.execute_reply.started":"2021-12-04T14:49:19.249659Z","shell.execute_reply":"2021-12-04T16:34:32.629311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot learning curves\nhistory_df = pd.DataFrame(history.history)\nhistory_df.loc[1:, ['loss', 'val_loss']].plot(title=\"Cross-entropy\")\nhistory_df.loc[1:, ['categorical_accuracy', 'val_categorical_accuracy']].plot(title=\"Accuracy\")\n\n# Expected accuracy on test set\nprint('Final accuracy on validation set:',\n      history_df.loc[len(history_df)-1,'val_categorical_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:32.631448Z","iopub.execute_input":"2021-12-04T16:34:32.631674Z","iopub.status.idle":"2021-12-04T16:34:33.363811Z","shell.execute_reply.started":"2021-12-04T16:34:32.631648Z","shell.execute_reply":"2021-12-04T16:34:33.362857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds = np.argmax(model.predict(X_valid), axis = 1)\ntrain_preds = np.argmax(model.predict(X_train), axis = 1)\n\nfig, axes = plt.subplots(1, 2, figsize = (18, 6))\n\ncm_train = confusion_matrix(np.argmax(y_train, axis = 1), train_preds)\ndisp = ConfusionMatrixDisplay(confusion_matrix = cm_train)\ndisp.plot(cmap = plt.cm.Blues, ax = axes[0])\n\ncm_val = confusion_matrix(np.argmax(y_valid, axis = 1), val_preds)\ndisp = ConfusionMatrixDisplay(confusion_matrix = cm_val)\ndisp.plot(cmap = plt.cm.Blues, ax = axes[1])\n\naxes[0].set_title(\"Training Set\"); axes[1].set_title(\"Validation Set\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:33.365316Z","iopub.execute_input":"2021-12-04T16:34:33.365799Z","iopub.status.idle":"2021-12-04T16:34:49.893673Z","shell.execute_reply.started":"2021-12-04T16:34:33.365746Z","shell.execute_reply":"2021-12-04T16:34:49.892708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"errors = (val_preds - np.argmax(y_valid, axis = 1) != 0)\n\npred_error = val_preds[errors]\nobserved_error = np.argmax(y_valid, axis = 1)[errors]\nimage_error = X_valid[errors]\nlen(pred_error)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:49.894744Z","iopub.execute_input":"2021-12-04T16:34:49.895008Z","iopub.status.idle":"2021-12-04T16:34:49.903550Z","shell.execute_reply.started":"2021-12-04T16:34:49.894978Z","shell.execute_reply":"2021-12-04T16:34:49.902433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(1, figsize=(15, 15))\nfig.suptitle(\"Errors in Validation\")\n\nrows = int(len(pred_error) ** 0.5) - 1\ncols = int(len(pred_error) / rows) + 1\n\nfor i in range(len(pred_error)):\n    \n    plt.subplot(rows, cols, i + 1)\n    plt.imshow(image_error[i], cmap = plt.cm.binary)\n    plt.axis(\"off\")\n    plt.title(f\"True Value: {observed_error[i]} \\nPrediction: {pred_error[i]}\")\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:49.905366Z","iopub.execute_input":"2021-12-04T16:34:49.906678Z","iopub.status.idle":"2021-12-04T16:34:50.870979Z","shell.execute_reply.started":"2021-12-04T16:34:49.906530Z","shell.execute_reply":"2021-12-04T16:34:50.869971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict labels of test data\npreds=model.predict(X_test)\n\n# Choose most likely class\npred_classes = np.argmax(preds,axis=1)\n\n# Save predictions to file\noutput = pd.DataFrame({'ImageId': test_index+1,\n                       'Label': pred_classes})\n\n# Check format\noutput.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:34:50.872687Z","iopub.execute_input":"2021-12-04T16:34:50.873026Z","iopub.status.idle":"2021-12-04T16:35:01.386603Z","shell.execute_reply.started":"2021-12-04T16:34:50.872983Z","shell.execute_reply":"2021-12-04T16:35:01.385733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.to_csv('submission.csv', index=False)\nprint(\"Finished!\")","metadata":{"execution":{"iopub.status.busy":"2021-12-04T16:35:01.387763Z","iopub.execute_input":"2021-12-04T16:35:01.388017Z","iopub.status.idle":"2021-12-04T16:35:01.450738Z","shell.execute_reply.started":"2021-12-04T16:35:01.387986Z","shell.execute_reply":"2021-12-04T16:35:01.449920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<hr>\n<br>\n<h3><p style=\"color:blue;\" text=\"align:center;\">Up vote if you thing the notebook was usefull!</p><h3>\n<hr>","metadata":{}}]}