{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h2>Welcome to a MNIST classic CNN with the new Mish activation function</h2> Warning, not actually using Mish yet, function has major speed problems!\n\nI'm doing a test of the new activation function on this fancy MNIST dataset. This Kernel is an adaptation of my original MNIST dataset one.\nYou can find the novel activation function here: [Mish paper](https://arxiv.org/abs/1908.08681)\n\nDon't worry I will try to explain everything along the way!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Just some standard python machine learning imports, we will be using keras from tensorflow\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd \n\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.layers import Activation\nfrom sklearn import metrics\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.layers import LeakyReLU\nfrom tensorflow.keras.optimizers import RMSprop\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Load the data from the csv files into pandas dataframes\ntrain = pd.read_csv('../input/Kannada-MNIST/train.csv')\ntest = pd.read_csv('../input/Kannada-MNIST/test.csv')\ndig = pd.read_csv('../input/Kannada-MNIST/Dig-MNIST.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract column 0 (the labels)\nlabels = train.iloc[:,0].values.astype('int32')\n\n# Extract the rest of the columns (the images)\nX_train = (train.iloc[:,1:].values).astype('float32')/255\n\n# Extract the testing images\nX_test = test.iloc[:,1:].values.astype('float32')/255\n\n# Do the same extraction for the Dig dataset\ndig_labels = dig.iloc[:,0].values.astype('float32')\ndig_val = dig.iloc[:,1:].values.astype('int32')/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Just a little bit of array shape validation\nprint(X_train.shape)\nprint(labels.shape)\n\nprint(X_test.shape)\n\nprint(dig_val.shape)\nprint(dig_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape continuous arrays into 28 by 28 greyscale images\nX_train = X_train.reshape(-1,28,28,1)\nX_test = X_test.reshape(-1,28,28,1)\ndig_images = dig_val.reshape(-1,28,28,1)\n\n# One hot enconde labels\ny_train = tf.keras.utils.to_categorical(labels) \n\n# Just a check for my mental sake heh\nprint(\"Check data\")\nprint(labels)\nprint(X_train[0].shape)\nprint(X_test[0].shape)\nprint(dig_images[0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# let's vizualise a single image\nfig = plt.figure()\nplt.imshow(X_train[6][:,:,0], cmap='gray', interpolation='none')\nplt.xticks([])\nplt.yticks([])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the input and output layer sizes\ninput_size = X_train.shape\nn_logits = y_train.shape[1]\n\nprint(\"Input: {}\".format(input_size))\nprint(\"Output: {}\".format(n_logits))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 50\nbatch_size = 1024\n#get validation data\n\nX_train, X_val, Y_train, Yval = train_test_split(X_train, y_train, train_size = 0.90)\n\n\ndatagen = ImageDataGenerator(rotation_range = 10,\n                           width_shift_range = 0.25,\n                           height_shift_range = 0.25,\n                           shear_range = 0.1,\n                           zoom_range = 0.25,\n                           horizontal_flip = False)\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here we define our keras model\nmodel = tf.keras.Sequential()\n\n# convolutions\nmodel.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3),padding='same',input_shape=input_size[1:]))\nmodel.add(tf.keras.layers.LeakyReLU(alpha=0.1))\nmodel.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=None))\nmodel.add(tf.keras.layers.Conv2D(64, kernel_size=(3, 3), padding='same'))\nmodel.add(tf.keras.layers.LeakyReLU(alpha=0.1))\nmodel.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(tf.keras.layers.Dropout(0.25))\nmodel.add(tf.keras.layers.Flatten())\n\n# head network\nmodel.add(tf.keras.layers.Dense(256))\nmodel.add(tf.keras.layers.LeakyReLU(alpha=0.1))\nmodel.add(tf.keras.layers.Dense(256))\nmodel.add(tf.keras.layers.LeakyReLU(alpha=0.1))\nmodel.add(tf.keras.layers.Dropout(0.5))\n# output\nmodel.add(tf.keras.layers.Dense(n_logits, activation='softmax'))\n\n\nes = EarlyStopping(monitor='val_loss', verbose=1, patience=10)\n\n# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\n\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='Adam',\n              metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit the model\nhist = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Yval),\n                              verbose = 1, steps_per_epoch=100, callbacks=[learning_rate_reduction, es])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_hist(history):\n    accuracy = history.history['accuracy']\n    val_accuracy = history.history['val_accuracy']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    epochs = range(len(accuracy))\n    plt.plot(epochs, accuracy, 'b', label='Training accuracy')\n    plt.plot(epochs, val_accuracy, 'r', label='Test accuracy')\n    plt.title('Accuracy')\n    plt.legend()\n    plt.show()\n    plt.figure()\n    plt.plot(epochs, loss, 'b', label='Training loss')\n    plt.plot(epochs, val_loss, 'r', label='Test loss')\n    plt.title('Loss')\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_hist(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dig_pred=model.predict_classes(dig_images)\nprint(metrics.accuracy_score(dig_pred, dig_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate predictions\npredictions = model.predict_classes(X_test, verbose=0)\npd.DataFrame({\"id\": list(range(0,len(predictions))), \"label\": predictions}).to_csv(\"submission.csv\", index=False, header=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}