{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import the libraries","metadata":{}},{"cell_type":"code","source":"# Import the libraries for data manipulation and visualisation\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pathlib\nimport os\n\n# Import the libraries to create a neural network\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten, MaxPool2D\nfrom tensorflow.keras.activations import relu, softmax\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import to_categorical\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nimport itertools\n\n\nsns.set(style='white', context='notebook', palette='deep')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-12T15:36:45.893366Z","iopub.execute_input":"2021-09-12T15:36:45.893788Z","iopub.status.idle":"2021-09-12T15:36:45.903185Z","shell.execute_reply.started":"2021-09-12T15:36:45.893751Z","shell.execute_reply":"2021-09-12T15:36:45.902191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the data with Pandas","metadata":{}},{"cell_type":"code","source":"# Load the data\n\ntrain_csv = pd.read_csv('../input/digit-recognizer/train.csv')\n\ntest_csv = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:09.011940Z","iopub.execute_input":"2021-09-12T15:27:09.012192Z","iopub.status.idle":"2021-09-12T15:27:14.912336Z","shell.execute_reply.started":"2021-09-12T15:27:09.012167Z","shell.execute_reply":"2021-09-12T15:27:14.911426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the head of the train_csv file ( first 5 rows)\n\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:14.913855Z","iopub.execute_input":"2021-09-12T15:27:14.914118Z","iopub.status.idle":"2021-09-12T15:27:14.947341Z","shell.execute_reply.started":"2021-09-12T15:27:14.914093Z","shell.execute_reply":"2021-09-12T15:27:14.946455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create the training dataset","metadata":{}},{"cell_type":"code","source":"# Split the training data into X (input) & Y (output) data.\n\nx_train = train_csv.drop(labels = ['label'], axis=1)\n\ny_train = train_csv['label']","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:14.948839Z","iopub.execute_input":"2021-09-12T15:27:14.949118Z","iopub.status.idle":"2021-09-12T15:27:15.058280Z","shell.execute_reply.started":"2021-09-12T15:27:14.949091Z","shell.execute_reply":"2021-09-12T15:27:15.057510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display the data and check for null values","metadata":{}},{"cell_type":"code","source":"# Count the data and check if it equaly seperated\n\ng = sns.countplot(y_train)\n\n\n# It is importent to set the style (see import libararies) of seaborn otherwite there will be no number on the X and Y axis.\ny_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.059541Z","iopub.execute_input":"2021-09-12T15:27:15.059983Z","iopub.status.idle":"2021-09-12T15:27:15.321146Z","shell.execute_reply.started":"2021-09-12T15:27:15.059940Z","shell.execute_reply":"2021-09-12T15:27:15.320444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for null values\n\nprint('x_train data')\nx_train.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.322170Z","iopub.execute_input":"2021-09-12T15:27:15.322618Z","iopub.status.idle":"2021-09-12T15:27:15.364911Z","shell.execute_reply.started":"2021-09-12T15:27:15.322579Z","shell.execute_reply":"2021-09-12T15:27:15.363794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('test data')\ntest_csv.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.366450Z","iopub.execute_input":"2021-09-12T15:27:15.366854Z","iopub.status.idle":"2021-09-12T15:27:15.387251Z","shell.execute_reply.started":"2021-09-12T15:27:15.366812Z","shell.execute_reply":"2021-09-12T15:27:15.386198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize the data ","metadata":{}},{"cell_type":"code","source":"x_train = x_train / 255.0\n\ntest_csv = test_csv / 255.0","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.390497Z","iopub.execute_input":"2021-09-12T15:27:15.390802Z","iopub.status.idle":"2021-09-12T15:27:15.499222Z","shell.execute_reply.started":"2021-09-12T15:27:15.390772Z","shell.execute_reply":"2021-09-12T15:27:15.498202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reshape the data","metadata":{}},{"cell_type":"code","source":"x_train = x_train.values.reshape(-1, 28, 28, 1)\n\ntest_csv = test_csv.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.500942Z","iopub.execute_input":"2021-09-12T15:27:15.501363Z","iopub.status.idle":"2021-09-12T15:27:15.506445Z","shell.execute_reply.started":"2021-09-12T15:27:15.501309Z","shell.execute_reply":"2021-09-12T15:27:15.505447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label encoding","metadata":{}},{"cell_type":"code","source":"y_train = to_categorical(y_train, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.507647Z","iopub.execute_input":"2021-09-12T15:27:15.507993Z","iopub.status.idle":"2021-09-12T15:27:15.518863Z","shell.execute_reply.started":"2021-09-12T15:27:15.507964Z","shell.execute_reply":"2021-09-12T15:27:15.517927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Split training and validation","metadata":{}},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(x_train,y_train, test_size = 0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.520186Z","iopub.execute_input":"2021-09-12T15:27:15.520503Z","iopub.status.idle":"2021-09-12T15:27:15.921342Z","shell.execute_reply.started":"2021-09-12T15:27:15.520476Z","shell.execute_reply":"2021-09-12T15:27:15.920408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create the model","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:15.922550Z","iopub.execute_input":"2021-09-12T15:27:15.922826Z","iopub.status.idle":"2021-09-12T15:27:16.264496Z","shell.execute_reply.started":"2021-09-12T15:27:15.922799Z","shell.execute_reply":"2021-09-12T15:27:16.263721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compile the model","metadata":{}},{"cell_type":"code","source":"optimizer = Adam(learning_rate = 0.001)\n\nmodel.compile(optimizer = optimizer, loss= 'categorical_crossentropy', metrics= ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:16.265555Z","iopub.execute_input":"2021-09-12T15:27:16.265984Z","iopub.status.idle":"2021-09-12T15:27:16.281119Z","shell.execute_reply.started":"2021-09-12T15:27:16.265942Z","shell.execute_reply":"2021-09-12T15:27:16.280416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create the ImageDataGenerator","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(x_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:16.282195Z","iopub.execute_input":"2021-09-12T15:27:16.282661Z","iopub.status.idle":"2021-09-12T15:27:16.369739Z","shell.execute_reply.started":"2021-09-12T15:27:16.282615Z","shell.execute_reply":"2021-09-12T15:27:16.368829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fit the model","metadata":{}},{"cell_type":"code","source":"history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=86),\n                              epochs = 5, validation_data = (x_val,y_val),\n                              verbose = 2, steps_per_epoch= 100\n                              )","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:27:16.370856Z","iopub.execute_input":"2021-09-12T15:27:16.371291Z","iopub.status.idle":"2021-09-12T15:30:30.888582Z","shell.execute_reply.started":"2021-09-12T15:27:16.371246Z","shell.execute_reply":"2021-09-12T15:30:30.887534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot the loss and accucary in a graph","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 1)\n\nax[0].plot(history.history['accuracy'], color = 'r', label = 'train_accuracy')\nax[0].plot(history.history['val_accuracy'], color = 'b', label = 'val_accuracy')\nlegend = ax[0].legend(loc='best', shadow=True)\n\n\nax[1].plot(history.history['loss'], color = 'r', label = 'train_loss')\nax[1].plot(history.history['val_loss'], color = 'b', label = 'val_loss')\nlegend = ax[1].legend(loc='best', shadow=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:30:30.889963Z","iopub.execute_input":"2021-09-12T15:30:30.890247Z","iopub.status.idle":"2021-09-12T15:30:31.358072Z","shell.execute_reply.started":"2021-09-12T15:30:30.890220Z","shell.execute_reply":"2021-09-12T15:30:31.357085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a confusion matrix","metadata":{}},{"cell_type":"code","source":"# Look at confusion matrix \n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model.predict(x_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-09-12T15:36:51.467277Z","iopub.execute_input":"2021-09-12T15:36:51.467875Z","iopub.status.idle":"2021-09-12T15:36:58.965590Z","shell.execute_reply.started":"2021-09-12T15:36:51.467824Z","shell.execute_reply":"2021-09-12T15:36:58.964617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}