{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import math, random\nimport tensorflow as tf\nfrom tensorflow.keras.optimizers import RMSprop\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.models import Sequential\n# from keras.layers import Conv2D, Dense,Flatten, MaxPool2D, Dropout, BatchNormalization, Reshape\n\nprint(tf.__version__)\n\nnp.random.seed(2019)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train_file = \"../input/Kannada-MNIST/train.csv\"\ndata_test_file = \"../input/Kannada-MNIST/test.csv\"\n\ndf_train = pd.read_csv(data_train_file)\ndf_test = pd.read_csv(data_test_file)\nsubmissions = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head(8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Note this returns numpy arrays\ndef get_features_labels(df):\n    # The first column is the label.\n    labels = df['label'].values\n    \n    # Select all columns except the first\n    features = df.values[:, 1:]/255\n    \n    return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_features, train_labels = get_features_labels(df_train)\ntest_features = df_test.values/255\n# test_features, test_labels = get_features_labels(df_test)\n\nprint(train_features.shape)\nprint(test_features.shape)\nprint(train_labels.shape)\nprint(\"\\n\")\n\ntest_features1 = np.delete(test_features, 0, 1)\nX_train = train_features.reshape(train_features.shape[0], 28, 28, 1)\nX_test = test_features1.reshape(test_features1.shape[0], 28, 28, 1)\n\nprint(f\"Shape of training set is {X_train.shape} now.\" )\nprint(f\"Shape of test set is {X_test.shape} now.\" )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Used to understand numpy.ndarray. Can be skipped\n# b = np.arange(20).reshape(4, 5)\n# print(f\"Ndarray b is {b}.\")\n# b_del = np.delete(b, 2, 1)\n# print(\"The remaining array is: \")\n# print(b_del)\n\n# c = np.arange(24).reshape(2, 3, 4)\n# print(f\"Ndarray c is {c}.\")\n# print(\"The selected array is: \")\n# print(c[:, :, 2])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defaults to showing data from the training set, \n# but we can provide the test data as well, and leave labels as None, to visualize test set\ndef display_by_index(index, features = X_train, labels = train_labels):\n    plt.figure()\n    \n    if labels is not None:\n        plt.title(f'Label: {labels[index]}')\n        _ = plt.imshow(features[index, :, :, 0], 'gray')\n    else:\n        plt.title(f'Sample: {index}')\n        _ = plt.imshow(np.reshape(features[index, :], (28,28)), 'gray')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize a training sample\ndisplay_by_index(168)\n\n# Visualize a test sample. Helps to modify parameters of ImageDataGenerator\n# This is an example of large rotation\ndisplay_by_index(226, features = X_test, labels=None)\n# This is an example of large shear\ndisplay_by_index(4604, features = X_test, labels=None)\ndisplay_by_index(random.randrange(X_test.shape[0]), features = X_test, labels = None)\ndisplay_by_index(random.randrange(X_test.shape[0]), features = X_test, labels = None)\ndisplay_by_index(random.randrange(X_test.shape[0]), features = X_test, labels = None)\ndisplay_by_index(random.randrange(X_test.shape[0]), features = X_test, labels = None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['label'].value_counts()\nprint(df_test.shape)\n\nY_train_1hot = tf.keras.utils.to_categorical(train_labels)\n\nprint(Y_train_1hot.shape)\nprint(\"\\n\")\n\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train_1hot, random_state = 42, test_size = 0.05)\n# X_train = X_train.reshape(X_train.shape[0], 784)\n# X_val = X_val.reshape(X_val.shape[0], 784)\nprint(X_train.shape)\nprint(Y_train.shape)\nprint(X_val.shape)\nprint(Y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_arch = {}\n\n# lr decay function\n# def lr_decay(epoch):\n    # return 0.01 * math.pow(0.8, epoch)\n\n# lr schedule callback\n# lr_decay_callback = tf.keras.callbacks.LearningRateScheduler(lr_decay, verbose=True)\n\nlearning_rate_reduction = ReduceLROnPlateau(monitor = 'val_acc', \n                                            patience = 3, # Reduce lr only when accuracy doesn't increase for 3 epochs\n                                            verbose = 1, \n                                            factor = 0.3, \n                                            min_lr = 0.00001)\n\n# Plot the decay rate\n# x = []\n# y = []\n# for i in range(1,10):\n    # y.append(lr_decay(i))\n    # x.append(i)\n# plt.plot(y, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Other models (not used)\n# model_arch['single_layer'] = [\n      # tf.keras.layers.Input(shape=(28*28,)),\n      # tf.keras.layers.Dense(10, activation='softmax')\n  # ]\n\n# model_arch['dnn'] = [\n      # tf.keras.layers.Input(shape=(28*28,)),\n      # tf.keras.layers.Dense(256, activation='sigmoid'),\n      # tf.keras.layers.Dense(64, activation='sigmoid'),\n      # tf.keras.layers.Dense(10, activation='softmax')\n  # ]\n\n# model_arch['dnn_relu'] = [\n      # tf.keras.layers.Input(shape=(28*28,)),\n      # tf.keras.layers.Dense(256, activation='relu'),\n      # tf.keras.layers.Dense(64, activation='relu'),\n      # tf.keras.layers.Dense(10, activation='softmax')\n  # ]\n\n# Add dropout\n# model_arch['dnn_relu_dropout'] = [\n      # tf.keras.layers.Input(shape=(28*28,)),\n      # tf.keras.layers.Dense(256, activation='relu'),\n      # tf.keras.layers.Dropout(0.25),\n      # tf.keras.layers.Dense(64, activation='relu'),\n      # tf.keras.layers.Dropout(0.25),\n      # tf.keras.layers.Dense(10, activation='softmax')\n  # ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate more images based on training set\ngenerated_data = ImageDataGenerator(\n                 rotation_range = 20,\n                 shear_range = 0.1,\n                 zoom_range = 0.1,\n                 width_shift_range = 0.1,\n                 height_shift_range = 0.1)\n\ngenerated_data.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build convolutional neural network\nmodel_arch['cnn'] = [\n      # tf.keras.layers.Reshape(input_shape=(28*28,), target_shape=(28, 28, 1)),\n      tf.keras.layers.Reshape(input_shape = (28, 28, 1), target_shape = (28, 28, 1)),\n      tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),\n      # tf.keras.layers.BatchNormalization(),\n      tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),\n      tf.keras.layers.BatchNormalization(),\n      tf.keras.layers.MaxPooling2D((2, 2), strides = 1),\n      tf.keras.layers.Dropout(0.5),\n    \n      tf.keras.layers.Conv2D(filters = 64, kernel_size = 3, activation='relu', padding='same'),\n      # tf.keras.layers.BatchNormalization(),\n      tf.keras.layers.Conv2D(filters = 64, kernel_size = 3, activation='relu', padding='same'),\n      tf.keras.layers.BatchNormalization(),\n      tf.keras.layers.MaxPooling2D((2, 2), strides = 2),\n      tf.keras.layers.Dropout(0.5),\n    \n      tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),\n      # tf.keras.layers.BatchNormalization(),\n      tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, activation='relu', padding='same'),\n      tf.keras.layers.BatchNormalization(),\n      tf.keras.layers.MaxPooling2D((2, 2), strides = 2),\n      tf.keras.layers.Dropout(0.4),\n    \n      tf.keras.layers.Flatten(),\n      tf.keras.layers.Dense(256, activation='relu'),\n      tf.keras.layers.Dropout(0.4),\n      tf.keras.layers.Dense(10, activation='softmax')\n      ]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_arch.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel = tf.keras.Sequential(model_arch['cnn'])\n# optimizer = 'sgd'\ninitial_learningrate=2e-3\nuser_optimizer = RMSprop(lr=initial_learningrate)\n# optimizer = 'adam'\n# optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n\n# We will now compile and print out a summary of our model\nmodel.compile(loss = 'categorical_crossentropy',\n              optimizer = user_optimizer,\n              metrics = ['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"User_batch_size = 256\nEPOCHS = 24","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# history = model.fit(train_features, train_labels_1hot, \n          # epochs=EPOCHS, \n          # batch_size=BATCH_SIZE, \n          # validation_split=0.2,\n          # callbacks=[lr_decay_callback],\n          # shuffle = True)\n\n# Use fit_generator function to train the model on generated images\nhistory = model.fit_generator(generated_data.flow(X_train, Y_train, batch_size = User_batch_size),\n                             epochs = EPOCHS, validation_data = (X_val, Y_val), \n                             # steps_per_epoch = X_train.shape[0] // User_batch_size,\n                             shuffle = True,  # steps_per_epoch or shuffle, pick one to use\n                             verbose = 1, callbacks = [learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['acc'], color='b', label=\"Training accuracy\")\nplt.plot(history.history['val_acc'], color='r', label=\"Validation accuracy\")\nplt.legend(loc='lower right', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predictions = model.predict_classes(X_test)\npredictions = model.predict(X_test)\n# submissions = pd.DataFrame({\"id\": list(range(0,len(predictions))),\n                         # \"label\": predictions})\npredictions = np.argmax(predictions, axis = 1)\nsubmissions['label'] = predictions\nsubmissions.to_csv(\"submission.csv\", index = False, header = True)\n\n# submissions.head(8)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}