{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils.np_utils import to_categorical\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nfrom keras.optimizers import RMSprop\n\n\n\n# visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt, matplotlib.image as mpimg\n%matplotlib inline\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load & Prepare Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#load train data \ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\n\n# sprate imgs from labels\ntrain_imgs = train.drop(['label'], axis=1)\ntrain_labels = train['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_imgs(imgs):\n    #reshape to (28,28) \n    imgs = np.reshape(imgs.to_numpy(), (-1,28,28))\n    #scale the values to be [0,1]\n    imgs = imgs/255.\n    \n    return imgs\n    \n    \ndef prepare_lbls(lbls):\n    #labels to categories\n    lbl_cats = to_categorical(lbls, num_classes = 10)\n    \n    return lbl_cats\n\n\ndef split_data(X, Y, tst_sz, val_sz=0):\n    # to train & tst\n    X_trn, X_tst, y_trn, y_tst = train_test_split(X, Y, test_size =tst_sz, random_state=1)\n    \n    # to train & test & validation\n    if val_sz != 0:\n        X_val, X_tst, y_val, y_tst = train_test_split(X_tst, y_tst, test_size =0.5, random_state=1)\n        return X_trn, X_val, X_tst, y_trn, y_val, y_tst\n    \n    return X_trn, X_tst, y_trn, y_tst\n\n\ndef prepare(imgs, lbls=0, tst_sz=0, val_sz=0, to_tensor=0):\n    # prepare imgs\n    X = prepare_imgs(imgs)\n    \n    # prepare labels if lbls has values\n    if type(lbls)==pd.Series:\n        Y = prepare_lbls(lbls)\n    \n    \n    # if we want to split the data \n    if tst_sz!=0:\n        # split to 2 parts\n        if val_sz==0:\n            X_trn, X_tst, y_trn, y_tst = split_data(X, Y, tst_sz, val_sz=0)\n            # to tensor\n            if to_tensor == 1:\n                X_trn = tf.convert_to_tensor(X_trn[...,None])\n                X_tst = tf.convert_to_tensor(X_tst[...,None])\n                y_trn = tf.convert_to_tensor(y_trn)\n                y_tst = tf.convert_to_tensor(y_tst)\n            return X, Y, X_trn, X_tst, y_trn, y_tst\n        # split to 3 parts\n        else:\n            X_trn, X_val, X_tst, y_trn, y_val, y_tst = split_data(X, Y, tst_sz, val_sz=1)\n            # to tensor\n            if to_tensor == 1:\n                X_trn = tf.convert_to_tensor(X_trn[...,None])\n                X_val = tf.convert_to_tensor(X_val[...,None])\n                X_tst = tf.convert_to_tensor(X_tst[...,None])\n                y_trn = tf.convert_to_tensor(y_trn)\n                y_val = tf.convert_to_tensor(y_val)\n                y_tst = tf.convert_to_tensor(y_tst)\n            return X, Y, X_trn, X_val, X_tst, y_trn, y_val, y_tst\n        \n    #if we will work with the total dataset\n    if to_tensor == 1:\n        X = tf.convert_to_tensor(X[...,None])\n        \n    if type(lbls)!=pd.Series:\n        return X\n    return X, Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare and split data\nX, Y, X_trn, X_val, X_tst, y_trn, y_val, y_tst = prepare(train_imgs, train_labels, tst_sz=0.2, val_sz=1, to_tensor=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def vis_sample(imgs, lbls):\n    fig = plt.figure(figsize=(10, 10))\n    fig.suptitle('Sample Data', fontsize=16)\n    for i in range(9):\n        ax = plt.subplot(3, 3, i + 1)\n        plt.imshow(np.reshape(imgs[i], (28,28)))\n        plt.title(lbls[i].argmax())\n        plt.axis(\"off\")\n        plt.show;\n        \ndef lbls_hist(lbls):\n    hist,bin_edges = np.histogram(lbls)\n\n    plt.figure(figsize=[10,8])\n    plt.bar(bin_edges[:-1], hist, width = 0.5, color='#0504aa',alpha=0.7)\n    plt.xlabel('Label',fontsize=15)\n    plt.ylabel('Frequency',fontsize=15)\n    plt.title('Labels Distribution Histogram',fontsize=15)\n    plt.show();\n    \ndef explore_data(imgs, lbls):\n    # shape and dtype\n    print(f'Data shape is: {train_imgs.shape} and the data type is {type(train_imgs)}')\n    \n    # sample data vis\n    vis_sample(imgs, lbls)\n    \n    # labels hist\n    lbls_hist(lbls.argmax(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"explore_data(X, Y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_aug(X):\n    print(\"In Data Genrator Step\")\n    datagen = ImageDataGenerator(\n            featurewise_center=False,  \n            samplewise_center=False,  \n            featurewise_std_normalization=False,\n            samplewise_std_normalization=False,  \n            zca_whitening=False,  \n            rotation_range=10, \n            zoom_range = 0.1, \n            width_shift_range=0.1,  \n            height_shift_range=0.1,  \n            horizontal_flip=False,  \n            vertical_flip=False)  \n\n    datagen.fit(X)\n    return datagen\n    \n\ndef create_model(input_shape):\n    print(\"In Model Creating Step\")\n    model = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(32, (5,5), activation='relu',input_shape=input_shape),\n        tf.keras.layers.Conv2D(32, (5,5), activation='relu'),\n        tf.keras.layers.MaxPooling2D(2,2),\n        tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n        tf.keras.layers.MaxPooling2D(2,2),\n        tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n        tf.keras.layers.MaxPooling2D(2,2),\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(128,activation= 'relu'),\n        tf.keras.layers.Dense(10,activation= 'softmax')\n    ])\n    \n    optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n\n    model.compile(optimizer,\n                  loss=\"categorical_crossentropy\",\n                  metrics=['accuracy'])\n\n    print(model.summary())\n    return model\n    \n\ndef fit_model(model, X_trn, y_trn, n_epochs, X_tst, X_val=0, y_val=0, val_split=0, datagen=0):\n    if datagen == 0:\n        # without aug\n        if val_split != 0:\n            # using val split \n            print(\"In Model Fitting Step (using val split)\")\n            history = model.fit(\n                X_trn, y_trn, validation_split =val_split, \n                epochs=n_epochs\n            )\n        else:\n            #using val set\n            print(\"In Model Fitting Step (using val set)\")\n            history = model.fit(\n                X_trn, y_trn, validation_data = (X_val, y_val), \n                epochs=n_epochs\n            )\n    else:\n        # using aug \n        if  type(X_val)==tf.python.framework.ops.EagerTensor:\n            print(\"In Model Fitting Step (using val set & fit generator)\")\n            history = model.fit_generator(\n                datagen.flow(X_trn, y_trn), validation_data = (X_val, y_val) ,\n                epochs=n_epochs\n            )\n        else :\n            print(\"In Model Fitting Step (without val & using fit generator)\")\n            history = model.fit_generator(\n                datagen.flow(X_trn, y_trn),\n                epochs=n_epochs\n            )\n        \n    print(\"In Model Prediction Step\")\n    y_preds = model.predict(X_tst) \n    return history, y_preds\n\n\ndef vis_accuracy(history, n_epochs):\n    print(\"In Accuracy Visualization Step\")\n    acc = history.history['accuracy']\n    val_acc = history.history['val_accuracy']\n\n    loss=history.history['loss']\n    val_loss=history.history['val_loss']\n\n    epochs_range = range(n_epochs)\n\n    plt.figure(figsize=(16, 8))\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs_range, acc, label='Training Accuracy')\n    plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n    plt.legend(loc='lower right')\n    plt.title('Training and Validation Accuracy')\n\n    plt.subplot(1, 2, 2)\n    plt.plot(epochs_range, loss, label='Training Loss')\n    plt.plot(epochs_range, val_loss, label='Validation Loss')\n    plt.legend(loc='upper right')\n    plt.title('Training and Validation Loss')\n    plt.show();\n    \n\ndef check_accuracy(y_true, y_preds):\n    print(f'The model accuracy is {accuracy_score(y_tst, (y_preds > 0.5))}.')\n    \n\ndef check_confusion(y_true, y_preds):\n    plt.figure(figsize=(16, 8))\n    plt.title('Confusion Matrix')\n    sns.heatmap(confusion_matrix(y_true.numpy().argmax(1), y_preds.argmax(1)))\n    plt.show();\n    \n\ndef run_model(X_trn, y_trn, n_epochs, X_tst, y_tst=0, X_val=0, y_val=0, val_split=0, aug=0, final_model =0):\n    #create model\n    model = create_model(X_trn.shape[1:])\n    \n    # create data augmentation layer\n    datagen = 0\n    if aug == 1:\n        datagen = data_aug(X_trn)\n    \n    #fit & predict\n    history, y_preds = fit_model(model, X_trn, y_trn, n_epochs, X_tst, X_val, y_val, val_split, datagen)\n    \n    #if it's not the final model \"we have true y\"\n    if final_model == 0:\n        #vis accuracy curve\n        vis_accuracy(history, n_epochs)\n        #check confusion matrix\n        check_confusion(y_tst, y_preds)\n        #check total accuracy\n        check_accuracy(y_tst, y_preds)\n    \n    return y_preds\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train using val split\ny_preds = run_model(X_trn, y_trn, 10, X_tst, y_tst, val_split=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tain using val & test sets\ntst_y_preds_aug = run_model(X_trn, y_trn, 10, X_tst, y_tst, X_val, y_val, val_split=0, aug=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#load test data\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare full training data \nX, y = prepare(train_imgs, train_labels, tst_sz=0, val_sz=0, to_tensor=1)\nX_test = prepare(test, to_tensor=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train in full train data and pridect on test data \ntest_y_preds = run_model(X, y, 30, X_test, aug=1, final_model=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_y_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\"ImageId\": list(range(1,len(test_y_preds)+1)),\n                         \"Label\": test_y_preds.argmax(1)})\n\nsubmission.to_csv(\"Submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}