{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This Python 3 environment comes with many helpful analytics libraries installed\nIt is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\nFor example, here's several helpful packages to load\n\nInput data files are available in the read-only \"../input/\" directory\nFor example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nYou can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \nYou can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-16T08:52:23.711905Z","iopub.execute_input":"2021-11-16T08:52:23.71238Z","iopub.status.idle":"2021-11-16T08:52:23.721405Z","shell.execute_reply.started":"2021-11-16T08:52:23.712318Z","shell.execute_reply":"2021-11-16T08:52:23.71928Z"}}},{"cell_type":"markdown","source":"# Import Dependencies","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Conv2D, Dense, MaxPooling2D, Input, Flatten\nfrom tensorflow.keras.metrics import Precision, Recall\n\nfrom matplotlib import pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:18.500702Z","iopub.execute_input":"2021-11-16T10:20:18.501161Z","iopub.status.idle":"2021-11-16T10:20:21.501713Z","shell.execute_reply.started":"2021-11-16T10:20:18.501077Z","shell.execute_reply":"2021-11-16T10:20:21.500774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CONSTANT PARAMETER","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 2000\nPREFETCH_SIZE = tf.data.AUTOTUNE\nNUM_CLASS = 10","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:21.506709Z","iopub.execute_input":"2021-11-16T10:20:21.507080Z","iopub.status.idle":"2021-11-16T10:20:21.511622Z","shell.execute_reply.started":"2021-11-16T10:20:21.507045Z","shell.execute_reply":"2021-11-16T10:20:21.510664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data exploration","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \ntrain_csv = pd.read_csv('../input/digit-recognizer/train.csv')\ntest_csv  = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:21.513508Z","iopub.execute_input":"2021-11-16T10:20:21.514116Z","iopub.status.idle":"2021-11-16T10:20:27.430117Z","shell.execute_reply.started":"2021-11-16T10:20:21.514071Z","shell.execute_reply":"2021-11-16T10:20:27.428878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:27.433145Z","iopub.execute_input":"2021-11-16T10:20:27.433804Z","iopub.status.idle":"2021-11-16T10:20:27.456251Z","shell.execute_reply.started":"2021-11-16T10:20:27.433767Z","shell.execute_reply":"2021-11-16T10:20:27.455377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:27.457819Z","iopub.execute_input":"2021-11-16T10:20:27.458085Z","iopub.status.idle":"2021-11-16T10:20:27.474502Z","shell.execute_reply.started":"2021-11-16T10:20:27.458045Z","shell.execute_reply":"2021-11-16T10:20:27.473572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train shape: {} Test shape: {}'.format(train_csv.shape, test_csv.shape))","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:27.475857Z","iopub.execute_input":"2021-11-16T10:20:27.476182Z","iopub.status.idle":"2021-11-16T10:20:27.486621Z","shell.execute_reply.started":"2021-11-16T10:20:27.476143Z","shell.execute_reply":"2021-11-16T10:20:27.485909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset preparation","metadata":{}},{"cell_type":"code","source":"def img_resize(img_csv):\n    \n    # Convert from \"pandas.core.frame.DataFrame\" to \"numpy.ndarray\"\n    img_array = img_csv.to_numpy() \n    \n    # Preprocessing image\n    img_array = img_array / 255.0\n    \n    # Reshape image size to 28x28 pixels\n    img   = np.reshape(img_array,(img_array.shape[0],28,28))\n    \n    return img\n\ndef preprocessing(img_csv, label_csv):\n    \n    # Do all image preprocessings (convert Pandas -> numpy, so on)\n    img = img_resize(img_csv)\n    \n    # Convert from \"pandas.core.frame.DataFrame\" to \"numpy.ndarray\"\n    label_array = label_csv.to_numpy()\n    \n    # Preprocessing label from number to categories or dictionary of classes\n    label = tf.keras.utils.to_categorical(label_array, num_classes=NUM_CLASS)\n    \n    # Convert all data into list in order to make dataset from tensor slices\n    img_dataset   = tf.data.Dataset.from_tensor_slices(list(img[:]))\n    label_dataset = tf.data.Dataset.from_tensor_slices(list(label[:]))\n    \n    return tf.data.Dataset.zip((img_dataset, label_dataset))\n\nlabels = train_csv['label']\nimages = train_csv.drop(['label'],axis=1)\n\ndataset = preprocessing(images,labels)\n\n# Build dataLoader pipeline\ntrain_data = dataset.cache()\ntrain_data = train_data.shuffle(10000)\ntrain_data = train_data.batch(BATCH_SIZE)\ntrain_data = train_data.prefetch(PREFETCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:27.487956Z","iopub.execute_input":"2021-11-16T10:20:27.488397Z","iopub.status.idle":"2021-11-16T10:20:35.641943Z","shell.execute_reply.started":"2021-11-16T10:20:27.488359Z","shell.execute_reply":"2021-11-16T10:20:35.641172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## For debugging","metadata":{}},{"cell_type":"code","source":"for test_input, label in train_data.as_numpy_iterator():\n    print(test_input.shape, label.shape)\n    print(label[0])\n    plt.imshow(test_input[0])\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:35.643013Z","iopub.execute_input":"2021-11-16T10:20:35.643247Z","iopub.status.idle":"2021-11-16T10:20:36.392666Z","shell.execute_reply.started":"2021-11-16T10:20:35.643219Z","shell.execute_reply":"2021-11-16T10:20:36.391889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model engineering","metadata":{}},{"cell_type":"code","source":"def make_CNN_model():\n    start  = Input(shape=(28,28,1), name='Input image')\n\n    # Preprocessing\n    #aug    = RandomFlip()(start)\n\n    # Bottle neck style\n    hidden = Conv2D(16, (1,5), activation='relu', padding='same')(start)\n    hidden = Conv2D(16, (5,1), activation='relu', padding='same')(hidden)\n    hidden = MaxPooling2D((2,2))(hidden)\n\n    hidden = Conv2D(32, (1,5), activation='relu', padding='same')(hidden)\n    hidden = Conv2D(32, (5,1), activation='relu', padding='same')(hidden)\n    hidden = MaxPooling2D((2,2))(hidden)\n    \n    # Flatten\n    hidden = Flatten()(hidden)\n    hidden = Dense(32, activation='relu')(hidden)\n\n    # Output\n    output = Dense(NUM_CLASS, activation='softmax')(hidden)\n\n    model = Model(inputs=[start], outputs=output, name='Kaggle_Digit_Recognizer')\n        \n    return model\n\nmodel = make_CNN_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:36.394153Z","iopub.execute_input":"2021-11-16T10:20:36.394724Z","iopub.status.idle":"2021-11-16T10:20:36.625607Z","shell.execute_reply.started":"2021-11-16T10:20:36.394674Z","shell.execute_reply":"2021-11-16T10:20:36.624791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model using custom training step function","metadata":{}},{"cell_type":"markdown","source":"## Build training step function","metadata":{}},{"cell_type":"code","source":"@tf.function\ndef train_step(batch):\n\n    with tf.GradientTape() as tape:\n        # Get image\n        X = batch[:1]\n        # Get label\n        y_ground = batch[1]\n        # Forward pass\n        y_predict = model(X, training=True)\n        # Calculate Loss\n        loss = loss_function(y_ground, y_predict)\n    \n    # Calculate gradients\n    grad = tape.gradient(loss, model.trainable_variables)\n\n    # Calculate updated weight and apply to model\n    opt.apply_gradients(zip(grad, model.trainable_variables))\n\n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:36.626948Z","iopub.execute_input":"2021-11-16T10:20:36.627186Z","iopub.status.idle":"2021-11-16T10:20:36.633239Z","shell.execute_reply.started":"2021-11-16T10:20:36.627156Z","shell.execute_reply":"2021-11-16T10:20:36.632400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training loop function","metadata":{}},{"cell_type":"code","source":"def train(model, data, EPOCHS):\n    \n    for epoch in range(1, EPOCHS+1):\n        print('\\n Epoch {}/{}'.format(epoch, EPOCHS))\n        progress_bar = tf.keras.utils.Progbar(len(data))\n        recall = Recall()\n        precision = Precision()\n\n        for idx, batch in enumerate(data):\n            \n            loss = train_step(batch)\n            y_predict = model.predict(batch[:1])\n            \n            recall.update_state(batch[1],y_predict)\n            precision.update_state(batch[1],y_predict)\n            progress_bar.update(idx+1)\n        print('Loss: {:.4f} Recall: {:.4f} Precision: {:.4f}'.format(loss.numpy(), recall.result().numpy(), precision.result().numpy()))","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:36.634367Z","iopub.execute_input":"2021-11-16T10:20:36.634586Z","iopub.status.idle":"2021-11-16T10:20:36.644157Z","shell.execute_reply.started":"2021-11-16T10:20:36.634558Z","shell.execute_reply":"2021-11-16T10:20:36.643161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# Setup loss function and optimizer\nopt           = tf.keras.optimizers.Adam() # Default learning_rate=0.001 epsilon=1e-7\nloss_function = tf.keras.losses.CategoricalCrossentropy()\n\nEPOCHS = 10\ntrain(model, train_data, EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:20:36.645174Z","iopub.execute_input":"2021-11-16T10:20:36.645522Z","iopub.status.idle":"2021-11-16T10:23:18.616563Z","shell.execute_reply.started":"2021-11-16T10:20:36.645494Z","shell.execute_reply":"2021-11-16T10:23:18.615640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test results with test.csv file","metadata":{}},{"cell_type":"code","source":"# Preprocessing test images from CSV file\ntest_data = img_resize(test_csv)\n\n# Predict test images\nresults = model.predict(test_data)","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:23:18.619071Z","iopub.execute_input":"2021-11-16T10:23:18.619319Z","iopub.status.idle":"2021-11-16T10:23:21.588567Z","shell.execute_reply.started":"2021-11-16T10:23:18.619283Z","shell.execute_reply":"2021-11-16T10:23:21.587624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print result\nprint(\"Result: {}\".format(results[0:10].round()))","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:23:21.589990Z","iopub.execute_input":"2021-11-16T10:23:21.590297Z","iopub.status.idle":"2021-11-16T10:23:21.596223Z","shell.execute_reply.started":"2021-11-16T10:23:21.590244Z","shell.execute_reply":"2021-11-16T10:23:21.595515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decode from category to its label\npred_test = [np.argmax(i) for i in results.round()]\nprint(pred_test[0:10])","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:23:21.597199Z","iopub.execute_input":"2021-11-16T10:23:21.597916Z","iopub.status.idle":"2021-11-16T10:23:21.658647Z","shell.execute_reply.started":"2021-11-16T10:23:21.597877Z","shell.execute_reply":"2021-11-16T10:23:21.657796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(pred_test, \n                          index=pd.Series(range(1,28001), name='ImageId'),\n                          columns=['Label'])\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:23:21.660001Z","iopub.execute_input":"2021-11-16T10:23:21.660508Z","iopub.status.idle":"2021-11-16T10:23:21.680208Z","shell.execute_reply.started":"2021-11-16T10:23:21.660461Z","shell.execute_reply":"2021-11-16T10:23:21.679626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"MNIST_prediction_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-16T10:23:40.913630Z","iopub.execute_input":"2021-11-16T10:23:40.914406Z","iopub.status.idle":"2021-11-16T10:23:40.953767Z","shell.execute_reply.started":"2021-11-16T10:23:40.914360Z","shell.execute_reply":"2021-11-16T10:23:40.952844Z"},"trusted":true},"execution_count":null,"outputs":[]}]}