{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import Library"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest_data = pd.read_csv(\"../input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"both train and test data consist of 784 pixel of image. but of course in the training data we found label column which we won't found in the test data"},{"metadata":{},"cell_type":"markdown","source":"# Exploratory Data Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Both train and test data doesn't have null values."},{"metadata":{},"cell_type":"markdown","source":"Now we will trying to see the image from that 784 pixel"},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image(pixel, label, index):\n    image2d = pixel.values.reshape(28,28)\n    plt.subplot('33%d' % (index))\n    plt.imshow(image2d, cmap=plt.cm.gray)\n    plt.title(label)\n\nplt.figure(figsize=(5,5))\nsample_image = train_data.sample(9).reset_index(drop=True)\n\nfor index, image in sample_image.iterrows():\n    label = image['label']\n    pixel = image.drop('label')\n    show_image(pixel, label, index)\n    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total all train data = %d \" % len(train_data))\nprint(\"Total all test data = %d \" % len(test_data), end='\\n============================\\n')\n\nlabels = train_data.sort_values('label', ascending=True).label.unique()\n\nfor i in labels:\n    print('Total data train in class %d = %d' % (i, len(train_data[train_data['label'] == i])))\n\nplt.figure(figsize=(10,5))\nsns.countplot(train_data['label'], palette='icefire')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.drop(columns=['label']).to_numpy().max()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing\n### Split data test and train"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\n\nX = train_data.drop(columns=['label']).values.reshape(train_data.shape[0], 28, 28, 1)\ny = to_categorical(train_data['label'], num_classes=10)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Agumentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(\n    rotation_range=10,\n    rescale=1./255,\n    zoom_range=0.1,\n#     shear_range=0.1,\n    width_shift_range=0.1,\n    height_shift_range=0.1\n)\n\nval_datagen = ImageDataGenerator(rescale=1./255)\n# val_datagen = ImageDataGenerator(\n#     rotation_range=20,\n#     rescale=1./255,\n#     zoom_range=0.2,\n# #     shear_range=0.1,\n#     width_shift_range=0.1,\n#     height_shift_range=0.1\n# )\n\ntrain_datagen.fit(X_train)\ntrain_datagen.fit(X_test)\n\ntrain_gen = train_datagen.flow(X_train, y_train, batch_size=32)\nval_gen = val_datagen.flow(X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelling\n### Build CNN Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout\n\n# model = Sequential([\n#     Conv2D(32, kernel_size=5, padding='Same', activation='relu', input_shape=(28,28,1)),\n#     Conv2D(32, kernel_size=5, padding='Same', activation='relu'),\n#     MaxPooling2D(2,2),\n#     Dropout(0.25),\n    \n#     Conv2D(64, kernel_size=3, padding='Same', activation='relu'),\n#     Conv2D(64, kernel_size=3, padding='Same', activation='relu'),\n#     MaxPooling2D(pool_size=(2,2), strides=(2,2)),\n#     Dropout(0.25),\n    \n#     Flatten(),\n#     Dense(256, activation='relu'),\n#     Dropout(0.5),\n#     Dense(10, activation='softmax')\n# ])\n\nmodel = Sequential([\n    Conv2D(64, kernel_size=3, activation='relu', input_shape=(28,28,1)),\n    Conv2D(32, kernel_size=3, activation='relu'),\n    MaxPooling2D(pool_size=(2,2)),\n    Flatten(),\n    Dense(10, activation='softmax')\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Compile Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.optimizers import RMSprop\n\nmodel.compile(\n    loss='categorical_crossentropy', \n    optimizer='adam', \n#     optimizer=tf.optimizers.Adam(),\n    metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\n# callbacks = [\n#     EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=10),\n#     ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1),\n#     ModelCheckpoint('model.h5', verbose=1, save_best_only=True, save_weights_only=True)\n# ]\n\ncallbacks = [\n    EarlyStopping(monitor='val_loss', mode='min', patience=10, verbose=1),\n    ReduceLROnPlateau(factor=0.1, patience=3, min_lr=0.00001, verbose=1),\n    ModelCheckpoint('model.h5', verbose=1, save_best_only=True, save_weights_only=True)\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    train_gen,\n    steps_per_epoch=len(X_train) // 32,\n    validation_data=val_gen,\n    validation_steps=len(X_test) // 32,\n    epochs=100,\n    verbose=1,\n    callbacks=callbacks\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loss, train_acc = model.evaluate(train_gen, verbose=0)\ntest_loss, test_acc = model.evaluate(val_gen, verbose=0)\nprint('Train acc: %.3f%%, Test acc: %.3f%%' % (train_acc*100, test_acc*100))\nprint('Train loss: %.3f%%, Test loss: %.3f%%' % (train_loss*100, test_loss*100))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot Accuracy Loss Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\ntrain_loss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(0,callbacks[0].stopped_epoch+1)\nprint(\"Total epochs = {}\".format(callbacks[0].stopped_epoch+1))\n\nplt.figure(figsize=(8, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs, train_acc, label='Training Accuracy')\nplt.plot(epochs, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs, train_loss, label='Training Loss')\nplt.plot(epochs, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ny_pred = model.predict(X_test)\ny_pred_class = np.argmax(y_pred, axis=1)\ny_true = np.argmax(y_test, axis=1)\n\ncm = confusion_matrix(y_true, y_pred_class)\nplt.figure(figsize=(8,8))\nsns.heatmap(cm, annot=True, linewidths=0.01, fmt='0')\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"Actual\")\nplt.title('Confusion Matrix')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"error = (y_pred_class - y_true != 0)\ny_pred_class_error = y_pred_class[error]\ny_pred_error = y_pred[error]\ny_true_error = y_true[error]\nX_test_error = X_test[error]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)), cmap=plt.cm.gray)\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n            \ny_pred_error_prob = np.max(y_pred_error, axis=1)\ntrue_prob_error = np.diagonal(np.take(y_pred_error, y_true_error, axis=1))\ndelta_pred_error = y_pred_error_prob - true_prob_error\nsorted_delta_error = np.argsort(delta_pred_error)\nmost_important_error = sorted_delta_error[-6:]\n\nplt.figure(figsize=(5,5))\ndisplay_errors(most_important_error, X_test_error, y_pred_class_error, y_true_error)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict Unseen Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict result\ntest_processed = test_data.values.reshape(test_data.shape[0],28,28,1).astype(\"float32\") / 255\nresults = model.predict(test_processed)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5, 5))\nsample_test = test_data.head(9)\nfor index, image_pixels in sample_test.iterrows():\n    label = results[index]\n    show_image(image_pixels, label, index)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"digit_output.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}