{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Digit recognizer with simple ConvNet\nFirst, import necessary libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport PIL\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/digit-recognizer'\ntrain_df = pd.read_csv(os.path.join(ROOT_DIR, 'train.csv'))\ntest_df = pd.read_csv(os.path.join(ROOT_DIR, 'test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train size: {}, test size: {}'.format(train_df.shape, test_df.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert DataFrame to numpy array\n\nfeature_train = train_df.to_numpy()[:,1:]\nlabel_train = train_df.to_numpy()[:,0]\n\nfeature_test = test_df.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Splitting training-validation set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_train, feature_val, label_train, label_val = train_test_split(feature_train, label_train, test_size=0.2, stratify=label_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train size: {}, validation size: {}'.format(len(label_train), len(label_val)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To use with CNN, convert our flatten features from format (N, M) to (N, H, W, C).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_train = feature_train.reshape(-1, 28, 28, 1)\nfeature_val = feature_val.reshape(-1, 28, 28, 1)\n\nfeature_test = feature_test.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize single image\ndef visualize(image):\n    plt.imshow(image[:,:,0], cmap='gray')\n    plt.show()\n    \n# visualize images from different classes\ndef visualize_samples(input_arr, target_arr, num_classes=10):\n    \"\"\"\n    Inputs:\n    - input_arr: our features array\n    - target_arr: our labels array\n    - num_classes: number of classes we want to visualize, default = 10 (maximum).\n    \"\"\"\n    assert (num_classes > 0 and num_classes <= 10), 'Number of classes must be in range [1-10]'\n    fig, axs = plt.subplots(1, num_classes, figsize=(1.5*num_classes, 2))\n    if num_classes == 1:\n        axs.imshow(input_arr[target_arr==0][0][:,:,0], cmap='gray')\n        axs.axis('off')\n        axs.set_title('Digit 0')\n    else:\n        for i in range(num_classes):\n            axs[i].imshow(input_arr[target_arr==i][0][:,:,0], cmap='gray')\n            axs[i].axis('off')\n            axs[i].set_title('Digit {}'.format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_samples(feature_train, label_train, 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data augmentation\nCreate a ImageDataGenerator object which rotates, shifts our images. You can add more transformations, or change the maximum ranges of below transformations.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a generator\ntrain_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n    rotation_range=25,\n    height_shift_range=3,\n    width_shift_range=3\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64 # use whichever batch size you want\n\ntrain_ds = train_datagen.flow(feature_train, label_train, batch_size=BATCH_SIZE)  # train_ds is a generator\n\n# val_ds and test_ds are Dataset objects\nval_ds = tf.data.Dataset.from_tensor_slices((feature_val, label_val)).batch(BATCH_SIZE).prefetch(1)\ntest_ds = tf.data.Dataset.from_tensor_slices(feature_test).batch(BATCH_SIZE).prefetch(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define model and train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"layers = [\n    tf.keras.layers.Conv2D(16, kernel_size=3, strides=1, padding='same', input_shape=(28,28,1)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.ReLU(),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(32, kernel_size=3, strides=1, padding='valid'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.ReLU(),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(64, kernel_size=5, strides=1, padding='valid'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.ReLU(),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.MaxPool2D(strides=2),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(512, activation='relu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(10, activation='softmax')\n]\n\nmodel = tf.keras.Sequential(layers=layers)\n\nlr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(1e-3, decay_steps=3000, decay_rate=0.95)\noptimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)\nmodel.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model.fit(train_ds, epochs=40, validation_data=val_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot training history of our model\ndef plot_history(history):\n    acc = history.history['accuracy']\n    val_acc = history.history['val_accuracy']\n\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    plt.figure(figsize=(8, 8))\n    plt.subplot(2, 1, 1)\n    plt.plot(acc, label='Training Accuracy')\n    plt.plot(val_acc, label='Validation Accuracy')\n    plt.legend(loc='lower right')\n    plt.ylabel('Accuracy')\n    plt.ylim([min(plt.ylim()),1])\n    plt.title('Training and Validation Accuracy')\n\n    plt.subplot(2, 1, 2)\n    plt.plot(loss, label='Training Loss')\n    plt.plot(val_loss, label='Validation Loss')\n    plt.legend(loc='upper right')\n    plt.ylabel('Cross Entropy')\n    plt.ylim([0,1.0])\n    plt.title('Training and Validation Loss')\n    plt.xlabel('epoch')\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Plot our learning curves","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Val accuracy seems to be slightly higher than train accuracy, which is due to data augmentation (and Dropout at train time). Overall, our simple model performs quite well and is not prone to overfitting.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Make predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test = np.argmax(model.predict(test_ds), axis=1)\nout_df = pd.DataFrame({'ImageId': np.arange(1, len(pred_test)+1), 'Label': pred_test})\nout_df.to_csv('output.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}