{"cells":[{"metadata":{"id":"AFijhuh_F0Xl","colab_type":"code","outputId":"f74c2dba-984a-42d3-9ab7-ba02c3e8f699","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nimport os\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom tensorflow.keras.preprocessing.image import img_to_array, ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler, ModelCheckpoint\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras import backend as K\n\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.metrics import classification_report\nfrom sklearn.feature_extraction.image import extract_patches_2d\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nmatplotlib.use('Agg')     \n%matplotlib inline\n\nimport pandas as pd\n\nimport numpy as np\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dir = '../input/Kannada-MNIST/'","execution_count":null,"outputs":[]},{"metadata":{"id":"iQWnqf2fQkcy","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"L = 10 # labels\nS = 28 # dimensions (H, W)\nC = 1  # depth","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load dataset"},{"metadata":{"id":"cj8pDsPR-M0e","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Load the Kannada MNIST training dataset downloaded from Kaggle\n# into a Pandas DataFrame\nX_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"id":"cKTIpMK7OeAR","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":253},"outputId":"589201dd-0b45-4217-a1cb-c75bfba54a04","trusted":true},"cell_type":"code","source":"# Plot pixel intensities\nX_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"z59BX2Xv-UqW","colab_type":"code","outputId":"4fc758f0-edd1-4474-852e-60f313fd19f7","colab":{"base_uri":"https://localhost:8080/","height":419},"trusted":true},"cell_type":"code","source":"# Extract dataset samples pixel intensities\nX = X_df.iloc[:, 1:]\nX = X.values \n\n# Extract labels\ny = X_df.iloc[:, 0]\ny = y.values\n\n# Plot labels\npd.DataFrame(y)\n\ndel X_df","execution_count":null,"outputs":[]},{"metadata":{"id":"rHY3EEM7Qe4P","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"6e34bbf1-f98f-43f8-8419-648e5ae9b44d","trusted":true},"cell_type":"code","source":"# Reshape dataset samples to (num_samples, height[rows], width[cols])\n# to do some plotting\nX = X.reshape(X.shape[0], S, S)\nX.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"G6wlrE6mOy0y","colab_type":"code","outputId":"5db220af-38d3-4a2b-a33a-4c3679bb6598","colab":{"base_uri":"https://localhost:8080/","height":281},"trusted":true},"cell_type":"code","source":"# Plot a sample of the dataset\nplt.imshow(X[420])\nplt.title(label='Image at position 420 (blaze it)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"1tWY2Cc0PHuF","colab_type":"code","outputId":"bc890154-c229-4d56-9f13-4211b1094393","colab":{"base_uri":"https://localhost:8080/","height":586},"trusted":true},"cell_type":"code","source":"# Plot more samples\n\nfig, ax = plt.subplots(\n    nrows=6,\n    ncols=5,\n    figsize=[6, 8]\n)\n\nfor index, axi in enumerate(ax.flat):\n    axi.imshow(X[index])\n    axi.set_title(f'Image #{index}')\n\nplt.tight_layout(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"hTYhdH_Q-fTz","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Normalize pixel intensities to range [0, 1]\nX = X.astype('float32') / 255.0","execution_count":null,"outputs":[]},{"metadata":{"id":"x2WE9X7EDidV","colab_type":"code","outputId":"88fc6c28-08ce-4b97-d277-480ef0ee59b0","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"# Reshape the samples from the dataset\n# to meet Keras requirements (num_samples, height, width, num_channels)\nX = X.reshape(-1, 28, 28, 1)\nX.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"5EkoHCr5-vCJ","colab_type":"code","outputId":"6158e1c2-2afb-47e1-ef10-b2e0747d6bd1","colab":{"base_uri":"https://localhost:8080/","height":419},"trusted":true},"cell_type":"code","source":"# One-hot encode labels from integers to vectors\ny = to_categorical(y, num_classes=L)\npd.DataFrame(y)","execution_count":null,"outputs":[]},{"metadata":{"id":"s5ZkEQ6cWLls","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"outputId":"7cf4bc2f-9f1d-4a84-e1eb-e078486edd7b","trusted":true},"cell_type":"code","source":"# Split the dataset into 80/20 train/test sets\n(X_train, X_ttest, y_train, y_ttest) = train_test_split(X, y, test_size = 0.2, random_state=42)\nprint(X_train.shape) # <- num of train samples\nprint(X_ttest.shape) # <- num of test  samples\n\ndel X","execution_count":null,"outputs":[]},{"metadata":{"id":"Twnw5D9k-zlE","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Data Augmentation: Set the image generator\n\ndaug = ImageDataGenerator(\n    featurewise_center=False,\n    samplewise_center=False,\n    featurewise_std_normalization=False,\n    samplewise_std_normalization=False,\n    zca_whitening=False,\n    rotation_range=12,\n    zoom_range = 0.3, \n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    horizontal_flip=False,\n    vertical_flip=False\n)","execution_count":null,"outputs":[]},{"metadata":{"id":"lt3KsDRHQHjw","colab_type":"text"},"cell_type":"markdown","source":"# Build the model"},{"metadata":{"id":"yyv1La7e-8O_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import BatchNormalization, Conv2D, MaxPooling2D, Activation, Flatten, Dropout, Dense, LeakyReLU\n\nclass CustomNet(object):\n    @staticmethod\n    def build(width, height, num_classes, depth=3):\n        model = Sequential()\n        input_shape = (height, width, depth)\n        chan_dim = -1\n        \n        # (Conv => LReLU => BN) * 3 => POOL => DO\n        model.add(Conv2D(64, (3, 3), padding='same', input_shape=input_shape))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(64, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(64, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        \n        # (Conv => LReLU => BN) * 3 => POOL => DO\n        model.add(Conv2D(96, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(96, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(96, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.2))\n        \n        # (Conv => LReLU => BN) * 3 => POOL => DO\n        model.add(Conv2D(128, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(128, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(128, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.2))\n        \n        # (Conv => LReLU => BN) * 3 => POOL => DO\n        model.add(Conv2D(256, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(256, (3, 3), padding='same'))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.2))\n        \n        # (FC => LReLU => BN) * 2 => DO\n        model.add(Flatten())\n        model.add(Dense(256))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization())\n        model.add(Dense(128))\n        model.add(LeakyReLU(alpha=0.1))\n        model.add(BatchNormalization())\n        model.add(Dropout(0.5))\n        \n        # Softmax\n        model.add(Dense(num_classes))\n        model.add(Activation('softmax'))\n        \n        print(model.summary())\n        \n        return model","execution_count":null,"outputs":[]},{"metadata":{"id":"H1gFq6w1_EPG","colab_type":"code","outputId":"d5abbc20-87c8-4dba-cc93-493fa8654a9e","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"net = CustomNet()\n\nmodel = net.build(\n    width=S,\n    height=S,\n    num_classes=L,\n    depth=C)","execution_count":null,"outputs":[]},{"metadata":{"id":"KTpNXCOv_MSN","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"num_epochs = 80","execution_count":null,"outputs":[]},{"metadata":{"id":"OMvmtL3nQ-Ci","colab_type":"text"},"cell_type":"markdown","source":"# Set the optimizer and hyperparameters"},{"metadata":{"id":"DBEwOLKt_PFC","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"init_lr = 0.002\npower = 2.0\n\nadam_opt = Adam(\n    lr=init_lr,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=1e-08,\n    decay=0.0\n)\n\n\ndef polynomial_decay(epoch):\n    max_epochs = num_epochs\n    \n    return init_lr * (1 - (epoch / float(max_epochs))) ** power","execution_count":null,"outputs":[]},{"metadata":{"id":"UooJ3CYmQwHy","colab_type":"code","outputId":"90ba7e87-1482-4b1e-8b33-85296b6a5ef7","colab":{"base_uri":"https://localhost:8080/","height":281},"trusted":true},"cell_type":"code","source":"# Plot polynomial decay\nx = np.linspace(0, num_epochs)\nfx = [init_lr * (1 - (i / float(num_epochs))) ** power for i in range(len(x))]\nplt.plot(x, fx)\nplt.title(label=f'Polynomial decay, power {power}')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"B7MF62w5RL3Y","colab_type":"text"},"cell_type":"markdown","source":"# Training time!"},{"metadata":{"id":"WTK8tGFp_asi","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Ignore checkpointing on Kaggle kernel\n\n'''\ncheckpointHandler = ModelCheckpoint(\n    os.path.join(base_dir, 'best_weights.hdf5'),\n    monitor='val_loss',\n    save_best_only=True,\n    verbose=1\n)\n'''\n\ncallbacks = [\n    LearningRateScheduler(polynomial_decay),\n    # checkpointHandler\n]","execution_count":null,"outputs":[]},{"metadata":{"id":"af9qwYmE_dVT","colab_type":"code","outputId":"2f14bdd2-d464-4cac-a780-e405bff8807e","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"batch_size = 128\n\nprint('# Compiling the model...')\nmodel.compile(\n    loss='categorical_crossentropy',\n    optimizer=adam_opt,\n    metrics=['accuracy']\n)\n\nprint('# Training the network...')\nh = model.fit_generator(\n    daug.flow(X_train, y_train, batch_size=batch_size),\n    validation_data=(X_ttest, y_ttest),\n    epochs=num_epochs,\n    steps_per_epoch=len(X_train) // batch_size,\n    callbacks=callbacks,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"id":"zkTCiSyBRb1n","colab_type":"text"},"cell_type":"markdown","source":"# Evaluate"},{"metadata":{"id":"oXQ4M6Lz_eAb","colab_type":"code","outputId":"e04285ff-2698-40aa-b892-e87b83898be0","colab":{"base_uri":"https://localhost:8080/","height":323},"trusted":true},"cell_type":"code","source":"label_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\nprint('Confusion matrix:')\npreds = model.predict(X_ttest, batch_size=batch_size)\nprint(classification_report(y_ttest.argmax(axis=1),\npreds.argmax(axis=1), target_names=label_names))","execution_count":null,"outputs":[]},{"metadata":{"id":"mAd1OXGtReYN","colab_type":"text"},"cell_type":"markdown","source":"# Visualize curves"},{"metadata":{"id":"xGB-cLnW_toq","colab_type":"code","outputId":"83aa415b-4ec7-4e4b-aa49-60b34ff93bc1","colab":{"base_uri":"https://localhost:8080/","height":462},"trusted":true},"cell_type":"code","source":"plt.style.use('ggplot')\nplt.figure(figsize=(10, 7))\nplt.plot(np.arange(0, num_epochs), h.history['loss'], label='train_loss')\nplt.plot(np.arange(0, num_epochs), h.history['val_loss'], label='val_loss')\nplt.plot(np.arange(0, num_epochs), h.history['accuracy'], label='train_accuracy')\nplt.plot(np.arange(0, num_epochs), h.history['val_accuracy'], label='val_accuracy')\n\nplt.title('Training Loss and Accuracy')\nplt.ylabel('Loss/Accuracy')\nplt.xlabel('Epoch #')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"NQ5OuM1ONtMV","colab_type":"text"},"cell_type":"markdown","source":"# Kaggle"},{"metadata":{"id":"7MI8GAR1IDXq","colab_type":"code","outputId":"bb9c3f00-6021-4710-f6fd-4c3a48a5c1a0","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"'''\nPushing a submission to Kaggle.\n\nFirst of, load the test set, normalize it and reshape it.\nThen, make predictions via the trained model and build the\nsubmission csv file.\n'''\n\nsub_X_test = pd.read_csv(os.path.join(base_dir, 'test.csv'))     # Load CSV\nsub_X_test = sub_X_test.drop('id', axis=1)                       # Drop the ID column\nsub_X_test = sub_X_test.iloc[:,:].values                         # Get raw pixel intensities\nsub_X_test = sub_X_test.reshape(sub_X_test.shape[0], S, S, C)    # Reshape to meet Keras requirements\nsub_X_test = sub_X_test / 255.0                                  # Normalize to range [0, 1]\n\npreds = model.predict_classes(sub_X_test, batch_size=batch_size)        # Make predictions and build the submission file\n\n# https://www.kaggle.com/c/Kannada-MNIST/discussion/110586\nid_col = np.arange(preds.shape[0])\nsubmission = pd.DataFrame({'id': id_col, 'label': preds})\nprint(pd.DataFrame(submission))\nsubmission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}