{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-09T18:53:01.890662Z","iopub.execute_input":"2022-11-09T18:53:01.891194Z","iopub.status.idle":"2022-11-09T18:53:01.975193Z","shell.execute_reply.started":"2022-11-09T18:53:01.891109Z","shell.execute_reply":"2022-11-09T18:53:01.974218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Import and Preprocessing","metadata":{}},{"cell_type":"code","source":"# import train set\ntrain = pd.read_csv(\"/kaggle/input/Kannada-MNIST/train.csv\")\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:53:05.233398Z","iopub.execute_input":"2022-11-09T18:53:05.234116Z","iopub.status.idle":"2022-11-09T18:53:08.304472Z","shell.execute_reply.started":"2022-11-09T18:53:05.234078Z","shell.execute_reply":"2022-11-09T18:53:08.303729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import test set\ntest = pd.read_csv(\"/kaggle/input/Kannada-MNIST/Dig-MNIST.csv\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:53:11.132807Z","iopub.execute_input":"2022-11-09T18:53:11.133130Z","iopub.status.idle":"2022-11-09T18:53:11.639627Z","shell.execute_reply.started":"2022-11-09T18:53:11.133094Z","shell.execute_reply":"2022-11-09T18:53:11.638904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split into input data and labels\ny_train_full = train.iloc[:, 0].to_numpy()\nx_train_full = train.iloc[:, 1:].to_numpy()\n\ny_test = test.iloc[:, 0].to_numpy()\nx_test = test.iloc[:, 1:].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:53:25.297295Z","iopub.execute_input":"2022-11-09T18:53:25.297566Z","iopub.status.idle":"2022-11-09T18:53:25.304948Z","shell.execute_reply.started":"2022-11-09T18:53:25.297537Z","shell.execute_reply":"2022-11-09T18:53:25.304113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# view shape of input data and labels\nprint(x_train_full.shape)\nprint(y_train_full.shape)\nprint(x_test.shape)\nprint(y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:53:44.779628Z","iopub.execute_input":"2022-11-09T18:53:44.779911Z","iopub.status.idle":"2022-11-09T18:53:44.785357Z","shell.execute_reply.started":"2022-11-09T18:53:44.779881Z","shell.execute_reply":"2022-11-09T18:53:44.784436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a look at one image\nimport matplotlib.pyplot as plt\n\none_image = x_train_full[50]\n\nplt.imshow(one_image.reshape(28, 28), cmap = \"gray\")\ny_train_full[50]","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:53:47.853166Z","iopub.execute_input":"2022-11-09T18:53:47.853434Z","iopub.status.idle":"2022-11-09T18:53:48.072472Z","shell.execute_reply.started":"2022-11-09T18:53:47.853406Z","shell.execute_reply":"2022-11-09T18:53:48.071727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scale and change data type of images\nx_train_full = x_train_full/255\n\nx_train_full = x_train_full.astype(\"float32\")\n\nx_test = x_test/255\n\nx_test = x_test.astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:54:06.472710Z","iopub.execute_input":"2022-11-09T18:54:06.473010Z","iopub.status.idle":"2022-11-09T18:54:06.721858Z","shell.execute_reply.started":"2022-11-09T18:54:06.472979Z","shell.execute_reply":"2022-11-09T18:54:06.720998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a validation set\nfrom sklearn.model_selection import train_test_split\n\nx_train, x_val, y_train, y_val = train_test_split(x_train_full, y_train_full, test_size = 0.1, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:54:11.764496Z","iopub.execute_input":"2022-11-09T18:54:11.764785Z","iopub.status.idle":"2022-11-09T18:54:12.481774Z","shell.execute_reply.started":"2022-11-09T18:54:11.764756Z","shell.execute_reply":"2022-11-09T18:54:12.480987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline Model","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense\n\n# baseline = Sequential([\n#     Dense(10, input_shape = (784, ), activation = \"softmax\")\n# ])\n\n# baseline.compile(loss = \"sparse_categorical_crossentropy\",\n#                  optimizer = \"adam\",\n#                  metrics = [\"accuracy\"])\n\n# history = baseline.fit(x_train, y_train, epochs = 10, validation_data = (x_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:54:50.085504Z","iopub.execute_input":"2022-11-09T18:54:50.085779Z","iopub.status.idle":"2022-11-09T18:54:51.573548Z","shell.execute_reply.started":"2022-11-09T18:54:50.085751Z","shell.execute_reply":"2022-11-09T18:54:51.572756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plot loss and accuracy curves\n# train_accuracy = history.history[\"accuracy\"]\n# val_accuracy = history.history[\"val_accuracy\"]\n\n# train_loss = history.history[\"loss\"]\n# val_loss = history.history[\"val_loss\"]\n\n# epochs = range(len(train_accuracy))\n\n# plt.plot(epochs, train_accuracy, '-b')\n# plt.plot(epochs, val_accuracy, '-g')\n# plt.title(\"Accuracy Plot\")\n# plt.figure()\n\n# plt.plot(epochs, train_loss, '-b')\n# plt.plot(epochs, val_loss, '-g')\n# plt.title(\"Loss Plot\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN\n[Learn TensorFlow and deep learning, without a Ph.D.](https://cloud.google.com/blog/products/ai-machine-learning/learn-tensorflow-and-deep-learning-without-a-phd)","metadata":{}},{"cell_type":"markdown","source":"## Basic Model","metadata":{}},{"cell_type":"code","source":"# reshape images\nx_train = x_train.reshape(-1, 28, 28, 1)\nx_val = x_val.reshape(-1, 28, 28, 1)\nx_test = x_test.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:54:24.313215Z","iopub.execute_input":"2022-11-09T18:54:24.313886Z","iopub.status.idle":"2022-11-09T18:54:24.318637Z","shell.execute_reply.started":"2022-11-09T18:54:24.313840Z","shell.execute_reply":"2022-11-09T18:54:24.317708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten\n\ncnn_one = Sequential([\n    Conv2D(filters = 32, kernel_size = (3, 3), input_shape = (28, 28, 1), padding = \"same\", activation = \"relu\"),\n    MaxPooling2D(),\n    Conv2D(filters = 64, kernel_size = (3, 3), padding = \"same\", activation = \"relu\"),\n    MaxPooling2D(),\n    Flatten(),\n    Dense(256, activation = \"relu\"),\n    Dense(10, activation = \"softmax\")\n])\n\ncnn_one.compile(loss = \"sparse_categorical_crossentropy\",\n           optimizer = \"adam\",\n           metrics = [\"accuracy\"])\n\ncnn_one.fit(x_train, y_train, epochs = 10, validation_data = (x_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:55:01.477919Z","iopub.execute_input":"2022-11-09T18:55:01.478206Z","iopub.status.idle":"2022-11-09T18:55:51.241909Z","shell.execute_reply.started":"2022-11-09T18:55:01.478177Z","shell.execute_reply":"2022-11-09T18:55:51.240837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic CNN With Augmentation, Checkpoints, and Dropout","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.layers import Dropout\n\n# instantiate ImageDataGenerator object\ntrain_datagen = ImageDataGenerator(\n                  rotation_range = 20,\n                  width_shift_range = 0.2,\n                  height_shift_range = 0.2,\n                  shear_range = 0.2,\n                  fill_mode = \"nearest\",\n                  zoom_range = 0.2,\n)\n\n# instantiate checkpoint objects\nes = EarlyStopping(monitor = 'val_loss',\n                   patience = 5,\n                   restore_best_weights = True,\n                   verbose = 0)\n\nreduce_lr = ReduceLROnPlateau(\n                    patience = 2)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:56:10.963358Z","iopub.execute_input":"2022-11-09T18:56:10.963721Z","iopub.status.idle":"2022-11-09T18:56:10.970502Z","shell.execute_reply.started":"2022-11-09T18:56:10.963675Z","shell.execute_reply":"2022-11-09T18:56:10.969491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_two = Sequential([\n                    Conv2D(filters = 32, kernel_size = (3,3), input_shape = (28, 28, 1), padding = \"same\", activation = \"relu\"),\n                    MaxPooling2D(),\n                    Dropout(0.2),\n                    Conv2D(filters = 64, kernel_size = (3,3), padding = \"same\", activation = \"relu\"),\n                    MaxPooling2D(),\n                    Flatten(),\n                    Dense(256, activation = \"relu\"),\n                    Dropout(0.2),\n                    Dense(10, activation = \"softmax\")\n])\n\ncnn_two.compile(loss = \"sparse_categorical_crossentropy\",\n              optimizer = \"adam\",\n              metrics = [\"accuracy\"])\n\ncnn_two.fit(train_datagen.flow(x_train, y_train), epochs = 1000, batch_size = 32, validation_data = (x_val, y_val), \n                      callbacks = [es, reduce_lr])","metadata":{"execution":{"iopub.status.busy":"2022-11-09T18:56:13.700449Z","iopub.execute_input":"2022-11-09T18:56:13.700718Z","iopub.status.idle":"2022-11-09T19:02:49.019349Z","shell.execute_reply.started":"2022-11-09T18:56:13.700687Z","shell.execute_reply":"2022-11-09T19:02:49.018747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using a Pre-trained Model\n\n[Keras guide to transfer learning & fine-tuning](https://keras.io/guides/transfer_learning/)\n\n[List of available models](https://keras.io/api/applications/#usage-examples-for-image-classification-models)","metadata":{}},{"cell_type":"code","source":"# reshape and resize images\n# add two additional channels\n\n# from tensorflow.image import grayscale_to_rgb\n# from tensorflow import convert_to_tensor\n\n# x_train = grayscale_to_rgb(convert_to_tensor(x_train))\n# x_val = grayscale_to_rgb(convert_to_tensor(x_val))\n\n# from tensorflow.image import resize\n\n# x_train = resize(x_train, [32, 32])\n# x_val = resize(x_val, [32, 32])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.applications import EfficientNetB0 # make sure your internet is on!!\n\n# base_model = EfficientNetB0(\n#     weights= 'imagenet', # Load weights pre-trained on ImageNet.\n#     input_shape = (32, 32, 3),  # minimum acceptable dimensions\n#     include_top = False)  # Do not include the ImageNet classifier at the top.\n\n# # freeze base model\n# #base_model.trainable = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.layers import GlobalAveragePooling2D, Input\n# from tensorflow.keras import Model\n\n# inputs = Input(shape=(32, 32, 3))\n# # We make sure that the base_model is running in inference mode here,\n# # by passing `training=False`. This is important for fine-tuning, as you will\n# # learn in a few paragraphs.\n# x = base_model(inputs, training=False)\n# # Convert features of shape `base_model.output_shape[1:]` to vectors\n# x = GlobalAveragePooling2D()(x)\n# outputs = Dense(10)(x)\n# cnn_pretrained = Model(inputs, outputs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cnn_pretrained.compile(loss = \"sparse_categorical_crossentropy\",\n#               optimizer = \"adam\",\n#               metrics = [\"accuracy\"])\n\n# history = cnn_pretrained.fit(x_train, y_train, epochs = 30, validation_data = (x_val, y_val),\n#                    callbacks = [es, reduce_lr])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## An example of when using a pre-trained model actually works!!\nhttps://www.kaggle.com/code/maksymshkliarevskyi/cassava-leaf-disease-best-keras-cnn","metadata":{}},{"cell_type":"markdown","source":"# Evaluate using test set","metadata":{}},{"cell_type":"code","source":"print(cnn_one.evaluate(x_test, y_test))\nprint(cnn_two.evaluate(x_test, y_test))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:04:30.342708Z","iopub.execute_input":"2022-11-09T19:04:30.343014Z","iopub.status.idle":"2022-11-09T19:04:32.028220Z","shell.execute_reply.started":"2022-11-09T19:04:30.342981Z","shell.execute_reply":"2022-11-09T19:04:32.027514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit \"Best\" Model Predictions","metadata":{}},{"cell_type":"code","source":"# Import necessary files\nsample_submit = pd.read_csv(\"/kaggle/input/Kannada-MNIST/sample_submission.csv\")\nreal_test = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\")\n\nsample_submit.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:04:40.483237Z","iopub.execute_input":"2022-11-09T19:04:40.483754Z","iopub.status.idle":"2022-11-09T19:04:40.872520Z","shell.execute_reply.started":"2022-11-09T19:04:40.483716Z","shell.execute_reply":"2022-11-09T19:04:40.871793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:05:00.818329Z","iopub.execute_input":"2022-11-09T19:05:00.818596Z","iopub.status.idle":"2022-11-09T19:05:00.836054Z","shell.execute_reply.started":"2022-11-09T19:05:00.818567Z","shell.execute_reply":"2022-11-09T19:05:00.835360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process final test data\n\nreal_test = real_test.iloc[:, 1:].to_numpy()\nreal_test = real_test/255\nreal_test = real_test.astype(\"float32\")\nreal_test = real_test.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:05:24.963448Z","iopub.execute_input":"2022-11-09T19:05:24.963723Z","iopub.status.idle":"2022-11-09T19:05:24.985396Z","shell.execute_reply.started":"2022-11-09T19:05:24.963692Z","shell.execute_reply":"2022-11-09T19:05:24.984376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Take a look at a few predictions\npredictions = cnn_two.predict(real_test)\nprint(predictions[:3])\n\npredictions = predictions.argmax(axis = 1)\n\npredictions[:3]","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:05:44.194250Z","iopub.execute_input":"2022-11-09T19:05:44.194534Z","iopub.status.idle":"2022-11-09T19:05:44.385375Z","shell.execute_reply.started":"2022-11-09T19:05:44.194498Z","shell.execute_reply":"2022-11-09T19:05:44.384644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prep for submission\nsample_submit[\"label\"] = predictions\nsample_submit.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:06:22.324384Z","iopub.execute_input":"2022-11-09T19:06:22.324649Z","iopub.status.idle":"2022-11-09T19:06:22.336028Z","shell.execute_reply.started":"2022-11-09T19:06:22.324620Z","shell.execute_reply":"2022-11-09T19:06:22.335204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make submission\nsample_submit.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T19:06:28.081047Z","iopub.execute_input":"2022-11-09T19:06:28.081632Z","iopub.status.idle":"2022-11-09T19:06:28.094987Z","shell.execute_reply.started":"2022-11-09T19:06:28.081597Z","shell.execute_reply":"2022-11-09T19:06:28.094146Z"},"trusted":true},"execution_count":null,"outputs":[]}]}