{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1- Hello Friends\n\n**Hello Future Engineers, Nice to meet you!**\n\nFirst we import our libraries that will we need.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, BatchNormalization, MaxPool2D, Flatten, Dense, Dropout\nimport warnings\n\nsns.set(style=\"darkgrid\")\n%matplotlib inline\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-04T06:08:12.941052Z","iopub.execute_input":"2021-06-04T06:08:12.941494Z","iopub.status.idle":"2021-06-04T06:08:14.888219Z","shell.execute_reply.started":"2021-06-04T06:08:12.941391Z","shell.execute_reply":"2021-06-04T06:08:14.887422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2- Prepare Data\nImport **train** and **test** data as dataframes.","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('../input/digit-recognizer/test.csv')\ndf_submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:14.889438Z","iopub.execute_input":"2021-06-04T06:08:14.8898Z","iopub.status.idle":"2021-06-04T06:08:18.861534Z","shell.execute_reply.started":"2021-06-04T06:08:14.889763Z","shell.execute_reply":"2021-06-04T06:08:18.860725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(-5)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:18.862922Z","iopub.execute_input":"2021-06-04T06:08:18.863263Z","iopub.status.idle":"2021-06-04T06:08:18.891744Z","shell.execute_reply.started":"2021-06-04T06:08:18.863226Z","shell.execute_reply":"2021-06-04T06:08:18.890883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's split the **label** column from **train** dataframe.","metadata":{}},{"cell_type":"code","source":"y_train = df_train['label']\nX_train = df_train.drop('label', axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:18.893524Z","iopub.execute_input":"2021-06-04T06:08:18.893926Z","iopub.status.idle":"2021-06-04T06:08:18.977231Z","shell.execute_reply.started":"2021-06-04T06:08:18.893877Z","shell.execute_reply":"2021-06-04T06:08:18.976193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The classes of data and their number of images.","metadata":{}},{"cell_type":"code","source":"sns.countplot(y_train)\ny_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:18.978797Z","iopub.execute_input":"2021-06-04T06:08:18.979454Z","iopub.status.idle":"2021-06-04T06:08:19.177065Z","shell.execute_reply.started":"2021-06-04T06:08:18.979411Z","shell.execute_reply":"2021-06-04T06:08:19.175971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's split **X_train** to **X_,train** and **X_validate**.","metadata":{}},{"cell_type":"code","source":"X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size=.1, shuffle=True)\nprint(\"X_train shape:\", X_train.shape)\nprint(\"y_train shape:\", y_train.shape)\nprint(\"X_validation shape:\", X_validation.shape)\nprint(\"y_validation shape:\", y_validation.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:19.180681Z","iopub.execute_input":"2021-06-04T06:08:19.180947Z","iopub.status.idle":"2021-06-04T06:08:19.394556Z","shell.execute_reply.started":"2021-06-04T06:08:19.180919Z","shell.execute_reply":"2021-06-04T06:08:19.393446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the size of image and its channel, batch size, number of epochs, and steps/epoch.","metadata":{}},{"cell_type":"code","source":"size  = 28\nchannels = 1\nbatch = 128\nepochs = 16\nsteps_per_epoch=  X_train.shape[0] // batch","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:19.396183Z","iopub.execute_input":"2021-06-04T06:08:19.396578Z","iopub.status.idle":"2021-06-04T06:08:19.401813Z","shell.execute_reply.started":"2021-06-04T06:08:19.396538Z","shell.execute_reply":"2021-06-04T06:08:19.400625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reshape the data to make images ready for **ImageDataGenerator**:\n> reshape(number of rows, img width, img height, channels)","metadata":{}},{"cell_type":"code","source":"X_train = X_train.values.reshape(X_train.shape[0], size, size, channels)\nX_validation = X_validation.values.reshape(X_validation.shape[0], size, size, channels)\nX_test = df_test.values.reshape(df_test.shape[0], size, size, channels)\n\nprint(\"X_train shape:\", X_train.shape)\nprint(\"X_validation shape:\", X_validation.shape)\nprint(\"X_test shape:\", X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:19.403208Z","iopub.execute_input":"2021-06-04T06:08:19.403639Z","iopub.status.idle":"2021-06-04T06:08:19.413006Z","shell.execute_reply.started":"2021-06-04T06:08:19.4036Z","shell.execute_reply":"2021-06-04T06:08:19.411736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3- Data Augmentation\nLet's make augmentation on training data and only scaling on validation and test data.","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1./255,\n                             zoom_range=0.2,\n                             width_shift_range=.2, height_shift_range=.2,\n                             rotation_range=30,\n                             brightness_range=[0.8, 1.2],\n                             horizontal_flip=True)\n\ndatagenRescale = ImageDataGenerator(rescale=1./255)\n\nX_train = datagen.flow(X_train, y_train, batch_size=batch)\n\nX_validation = datagenRescale.flow(X_validation, y_validation)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:19.41532Z","iopub.execute_input":"2021-06-04T06:08:19.415765Z","iopub.status.idle":"2021-06-04T06:08:19.481204Z","shell.execute_reply.started":"2021-06-04T06:08:19.415725Z","shell.execute_reply":"2021-06-04T06:08:19.480223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nfor i in range(9):\n    plt.subplot(3, 3, i+1)\n    for X_batch, Y_batch in X_train:\n        image = X_batch[i]\n        plt.imshow(image, cmap='gray')\n        plt.xlabel(Y_batch[i])\n        break\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:19.482677Z","iopub.execute_input":"2021-06-04T06:08:19.483075Z","iopub.status.idle":"2021-06-04T06:08:21.391938Z","shell.execute_reply.started":"2021-06-04T06:08:19.483033Z","shell.execute_reply":"2021-06-04T06:08:21.391028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4- Setup Callbacks","metadata":{}},{"cell_type":"code","source":"checkpoint_filepath = 'digit-recognizer-best-model.hdf5'\n\noptimizer = Adam(learning_rate=0.001)\ncallback_checkpoint = ModelCheckpoint(filepath=checkpoint_filepath, save_weights_only=False, monitor='val_accuracy', mode='max', save_best_only=True)\ncallback_learningrate = ReduceLROnPlateau(monitor='accuracy', mode='max', min_delta=0.01, patience=3, factor=.05, verbose=1)\n\ncallbacks = [callback_checkpoint, callback_learningrate]","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:21.393397Z","iopub.execute_input":"2021-06-04T06:08:21.394Z","iopub.status.idle":"2021-06-04T06:08:21.40159Z","shell.execute_reply.started":"2021-06-04T06:08:21.393953Z","shell.execute_reply":"2021-06-04T06:08:21.400733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5- Design the Neural Network","metadata":{}},{"cell_type":"code","source":"Model = Sequential([Conv2D(filters=32,  kernel_size=(3,3), activation=\"relu\", input_shape=(size,size,channels)),\n                    BatchNormalization(),\n                    MaxPool2D(2,2, padding='same'),\n                    Dropout(0.25),\n                    \n                    Conv2D(filters=128,  kernel_size=(3,3), activation=\"relu\"),\n                    BatchNormalization(),\n                    MaxPool2D(2,2, padding='same'),\n                    Dropout(0.25),\n                 \n                    Conv2D(filters=512,  kernel_size=(3,3), activation=\"relu\"),\n                    BatchNormalization(),\n                    MaxPool2D(2,2, padding='same'),\n                    Dropout(0.25),\n                    \n                    \n                    Flatten(),\n                    BatchNormalization(),\n\n                    \n                    Dense(units=4096, activation=\"relu\"), \n                    BatchNormalization(),\n                    Dropout(0.5),\n                                  \n                    Dense(units=256, activation=\"relu\"),\n                    BatchNormalization(),\n                    Dropout(.25),\n                    \n                    \n                    Dense(units=10, activation=\"softmax\"),\n])\n\n\nModel.compile(optimizer=optimizer, loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n\n\nModel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:21.403083Z","iopub.execute_input":"2021-06-04T06:08:21.403524Z","iopub.status.idle":"2021-06-04T06:08:22.374746Z","shell.execute_reply.started":"2021-06-04T06:08:21.403472Z","shell.execute_reply":"2021-06-04T06:08:22.373814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6- Let's train","metadata":{}},{"cell_type":"code","source":"history = Model.fit(X_train, validation_data=X_validation, epochs=epochs, steps_per_epoch=steps_per_epoch, callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:08:22.377745Z","iopub.execute_input":"2021-06-04T06:08:22.37803Z","iopub.status.idle":"2021-06-04T06:24:27.888942Z","shell.execute_reply.started":"2021-06-04T06:08:22.378002Z","shell.execute_reply":"2021-06-04T06:24:27.884044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(len(acc))\n\nplt.figure(figsize=(15, 15))\nplt.subplot(2, 2, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(2, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:24:27.890662Z","iopub.status.idle":"2021-06-04T06:24:27.891772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7- Let's evaluate","metadata":{}},{"cell_type":"code","source":"score = Model.evaluate(X_validation) \n\nprint('Test loss:', score[0]) \nprint('Test accuracy:', score[1])","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:24:27.893329Z","iopub.status.idle":"2021-06-04T06:24:27.895257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8- Let's Check","metadata":{}},{"cell_type":"code","source":"X_test = X_test / 255.0\ny_pred = np.argmax(Model.predict(X_test),axis = 1) ","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:24:27.897042Z","iopub.status.idle":"2021-06-04T06:24:27.898253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor i in range(9):\n    plt.subplot(3,3,i+1)\n    plt.imshow(X_test[i],cmap='gray')\n    plt.xlabel(y_pred[i])\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:24:27.90613Z","iopub.status.idle":"2021-06-04T06:24:27.907035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 9- Let's Submit","metadata":{}},{"cell_type":"code","source":"df_submission['Label'] = y_pred\ndf_submission.head(-5)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:24:27.908245Z","iopub.status.idle":"2021-06-04T06:24:27.909267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False)\nsubmission = pd.read_csv('./submission.csv')\nsubmission.head(-5)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T06:24:27.910585Z","iopub.status.idle":"2021-06-04T06:24:27.911302Z"},"trusted":true},"execution_count":null,"outputs":[]}]}