{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport tensorflow as tf\nprint(tf.__version__)\n\nnp.random.seed(2019)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load and explore data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"data_train_file = \"../input/Kannada-MNIST/train.csv\"\ndata_test_file = \"../input/Kannada-MNIST/test.csv\"\n\ndf_train = pd.read_csv(data_train_file)\ndf_test = pd.read_csv(data_test_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Notice data is shifted to the left by one column, since the label is missing\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocess data into numpy"},{"metadata":{},"cell_type":"markdown","source":"I've created a helper function to take care of processing the features into numpy arrays, but there are many options here."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Note this returns numpy arrays\ndef get_features_labels(df):\n    # The first column is the label.\n    labels = df['label'].values\n    \n    # Select all columns except the first\n    features = df.values[:, 1:]/255\n    \n    return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_features_ids(df):\n    # The first column is the label.\n    labels = df['id'].values\n    \n    # Select all columns except the first\n    features = df.values[:, 1:]/255\n    \n    return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_features, train_labels = get_features_labels(df_train)\n\ntest_features, test_labels = get_features_ids(df_test)\n\n# test_features = df_test.values/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Confirm that the shape is what we expect: 42k in train, 28k in test, with 784 pixels per row"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_features.shape)\nprint(test_features.shape)\nprint(train_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize the numbers\nA helper function for visualizing a specific row"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defaults to showing data from the training set, \n# but we can provide the test data as well, and leave labels as None, to visualize test set\ndef display_by_index(index, features=train_features, labels=train_labels):\n    plt.figure()\n    \n    if labels is not None:\n        plt.title(f'Label: {labels[index]}')\n        \n    _ = plt.imshow(np.reshape(features[index, :], (28,28)), 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize a training sample\ndisplay_by_index(221)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize a test sample\ndisplay_by_index(221, features=test_features, labels=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dataset looks fairly balanced. No need to do additional work here."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Turning labels into 1-hot encoding transforms the shape from 1 column to 10 columns"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels_1hot = tf.keras.utils.to_categorical(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_labels_1hot.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels_1hot","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating the model \nFor this one we use a deep neural net, expecting \"okay\" results, but nothing spectacular.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_arch = {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_arch['single_layer'] = [\n      tf.keras.layers.Input(shape=(28*28,)),\n      tf.keras.layers.Dense(10, activation='softmax')\n  ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_arch['dnn'] = [\n      tf.keras.layers.Input(shape=(28*28,)),\n      tf.keras.layers.Dense(200, activation='sigmoid'),\n      tf.keras.layers.Dense(60, activation='sigmoid'),\n      tf.keras.layers.Dense(10, activation='softmax')\n  ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_arch['dnn_relu'] = [\n      tf.keras.layers.Input(shape=(28*28,)),\n      tf.keras.layers.Dense(200, activation='relu'),\n      tf.keras.layers.Dense(60, activation='relu'),\n      tf.keras.layers.Dense(10, activation='softmax')\n  ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Learning rate decay"},{"metadata":{"trusted":true},"cell_type":"code","source":"# lr decay function\ndef lr_decay(epoch):\n    return 0.01 * math.pow(0.6, epoch)\n\n# lr schedule callback\nlr_decay_callback = tf.keras.callbacks.LearningRateScheduler(lr_decay, verbose=True)\n\n# Plot the decay rate\nx = []\ny = []\nfor i in range(1,10):\n    y.append(lr_decay(i))\n    x.append(i)\nplt.plot(x, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Add dropout\nmodel_arch['dnn_relu_dropout'] = [\n      tf.keras.layers.Input(shape=(28*28,)),\n      tf.keras.layers.Dense(200, activation='relu'),\n      tf.keras.layers.Dropout(0.25),\n      tf.keras.layers.Dense(60, activation='relu'),\n      tf.keras.layers.Dropout(0.25),\n      tf.keras.layers.Dense(10, activation='softmax')\n  ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CNN\nmodel_arch['cnn'] = [\n      tf.keras.layers.Reshape(input_shape=(28*28,), target_shape=(28, 28, 1)),\n      tf.keras.layers.Conv2D(kernel_size=3, filters=12, activation='relu', padding='same'),\n      tf.keras.layers.Conv2D(kernel_size=6, filters=24, activation='relu', padding='same', strides=2),\n      tf.keras.layers.Conv2D(kernel_size=6, filters=32, activation='relu', padding='same', strides=2),\n#     modify\n       tf.keras.layers.Conv2D(kernel_size=6, filters=32, activation='relu', padding='same', strides=2),\n    \n      tf.keras.layers.Flatten(),\n      tf.keras.layers.Dense(200, activation='relu'),\n      tf.keras.layers.Dropout(0.25),\n      tf.keras.layers.Dense(10, activation='softmax')\n  ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Choose model architecture and compile"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_arch.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential(model_arch['cnn'])\n\n# optimizer = 'sgd'\noptimizer = 'adam'\n# optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n\n# We will now compile and print out a summary of our model\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=optimizer,\n              metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model\nAdjust the hyper params as needed."},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE=128\nEPOCHS=5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_features, train_labels_1hot, \n          epochs=EPOCHS, \n          batch_size=BATCH_SIZE, \n          validation_split=0.2)\n#           ,callbacks=[lr_decay_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['acc'], color='b', label=\"Training accuracy\")\nplt.plot(history.history['val_acc'], color='r', label=\"Validation accuracy\")\nplt.legend(loc='lower right', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict_classes(test_features)\n\nsubmissions=pd.DataFrame({\"Id\": list(range(0,len(predictions))),\n                         \"Label\": predictions})\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Spot-checking some values"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(200,210):\n    display_by_index(i, features=test_features, labels=submissions[\"Label\"].values)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}