{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-01T04:42:05.275706Z","iopub.execute_input":"2023-02-01T04:42:05.276025Z","iopub.status.idle":"2023-02-01T04:42:05.311773Z","shell.execute_reply.started":"2023-02-01T04:42:05.275951Z","shell.execute_reply":"2023-02-01T04:42:05.310936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom tensorflow.keras.optimizers import Adam \nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import (Conv2D, MaxPooling2D, Flatten, Dense, \n                          Dropout, Rescaling, RandomFlip, RandomRotation, BatchNormalization)\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import RMSprop\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:05.313168Z","iopub.execute_input":"2023-02-01T04:42:05.313473Z","iopub.status.idle":"2023-02-01T04:42:13.084983Z","shell.execute_reply.started":"2023-02-01T04:42:05.313445Z","shell.execute_reply":"2023-02-01T04:42:13.083918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = np.loadtxt('../input/digit-recognizer/train.csv', delimiter=',', skiprows=1)\ntest = np.loadtxt('../input/digit-recognizer/test.csv', delimiter=',', skiprows=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:13.087764Z","iopub.execute_input":"2023-02-01T04:42:13.088656Z","iopub.status.idle":"2023-02-01T04:42:44.197571Z","shell.execute_reply.started":"2023-02-01T04:42:13.088615Z","shell.execute_reply":"2023-02-01T04:42:44.196530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = train[:, 0]\n\ntrain_img = np.resize(train[:, 1:], (train.shape[0], 28, 28))\ntest_img = np.resize(test, (test.shape[0], 28, 28))","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:44.200037Z","iopub.execute_input":"2023-02-01T04:42:44.200466Z","iopub.status.idle":"2023-02-01T04:42:44.420352Z","shell.execute_reply.started":"2023-02-01T04:42:44.200391Z","shell.execute_reply":"2023-02-01T04:42:44.419281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let’s visualize few data from training","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(20, 10))\nfor i, img in enumerate(train_img[0:5], 1):\n    subplot = fig.add_subplot(1, 7, i)\n    plt.imshow(img, cmap='gray');\n    subplot.set_title('%s' % train_label[i - 1]);","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:44.421832Z","iopub.execute_input":"2023-02-01T04:42:44.422934Z","iopub.status.idle":"2023-02-01T04:42:44.944323Z","shell.execute_reply.started":"2023-02-01T04:42:44.422903Z","shell.execute_reply":"2023-02-01T04:42:44.943321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check disbalance target classes","metadata":{}},{"cell_type":"code","source":"unique, counts = np.unique(train_label, return_counts=True)\nprint(unique, counts)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:44.945892Z","iopub.execute_input":"2023-02-01T04:42:44.946258Z","iopub.status.idle":"2023-02-01T04:42:44.954923Z","shell.execute_reply.started":"2023-02-01T04:42:44.946223Z","shell.execute_reply":"2023-02-01T04:42:44.953817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(unique)):\n    print(f'Digit {i}: {counts[i] / len(train_label) * 100} %')","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:44.956496Z","iopub.execute_input":"2023-02-01T04:42:44.957150Z","iopub.status.idle":"2023-02-01T04:42:44.965641Z","shell.execute_reply.started":"2023-02-01T04:42:44.957107Z","shell.execute_reply":"2023-02-01T04:42:44.964440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=train_label)\nplt.xlabel('Digits')\nplt.show;","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:44.967366Z","iopub.execute_input":"2023-02-01T04:42:44.967829Z","iopub.status.idle":"2023-02-01T04:42:45.190332Z","shell.execute_reply.started":"2023-02-01T04:42:44.967697Z","shell.execute_reply":"2023-02-01T04:42:45.189412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reacale images","metadata":{}},{"cell_type":"code","source":"rescale = Sequential([\n  Rescaling(1./255)\n])","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:43:50.737394Z","iopub.execute_input":"2023-02-01T04:43:50.738405Z","iopub.status.idle":"2023-02-01T04:43:50.747564Z","shell.execute_reply.started":"2023-02-01T04:43:50.738356Z","shell.execute_reply":"2023-02-01T04:43:50.746554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = rescale(train_img[0])\n_ = plt.imshow(result)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:44:03.377678Z","iopub.execute_input":"2023-02-01T04:44:03.378090Z","iopub.status.idle":"2023-02-01T04:44:03.582497Z","shell.execute_reply.started":"2023-02-01T04:44:03.378050Z","shell.execute_reply":"2023-02-01T04:44:03.581390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Min and max pixel values:\", result.numpy().min(), result.numpy().max())","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:49.273728Z","iopub.execute_input":"2023-02-01T04:42:49.274543Z","iopub.status.idle":"2023-02-01T04:42:49.280163Z","shell.execute_reply.started":"2023-02-01T04:42:49.274501Z","shell.execute_reply":"2023-02-01T04:42:49.278951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image augmentation ","metadata":{}},{"cell_type":"code","source":"data_augmentation = Sequential([\n  RandomFlip(\"horizontal_and_vertical\"),\n  RandomRotation(0.2),\n])","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:49.281563Z","iopub.execute_input":"2023-02-01T04:42:49.282387Z","iopub.status.idle":"2023-02-01T04:42:49.304199Z","shell.execute_reply.started":"2023-02-01T04:42:49.282346Z","shell.execute_reply":"2023-02-01T04:42:49.303347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = tf.expand_dims(train_img[0], 0)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:49.305446Z","iopub.execute_input":"2023-02-01T04:42:49.305993Z","iopub.status.idle":"2023-02-01T04:42:49.311563Z","shell.execute_reply.started":"2023-02-01T04:42:49.305960Z","shell.execute_reply":"2023-02-01T04:42:49.310650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor i in range(9):\n  augmented_image = data_augmentation(image)\n  ax = plt.subplot(3, 3, i + 1)\n  plt.imshow(augmented_image[0])\n  plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:49.313018Z","iopub.execute_input":"2023-02-01T04:42:49.313636Z","iopub.status.idle":"2023-02-01T04:42:49.972301Z","shell.execute_reply.started":"2023-02-01T04:42:49.313602Z","shell.execute_reply":"2023-02-01T04:42:49.971393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN","metadata":{}},{"cell_type":"code","source":"Y_train = to_categorical(train_label, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:49.974226Z","iopub.execute_input":"2023-02-01T04:42:49.975259Z","iopub.status.idle":"2023-02-01T04:42:49.981667Z","shell.execute_reply.started":"2023-02-01T04:42:49.975220Z","shell.execute_reply":"2023-02-01T04:42:49.980744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_img, Y_train, test_size = 0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:49.983067Z","iopub.execute_input":"2023-02-01T04:42:49.983512Z","iopub.status.idle":"2023-02-01T04:42:50.079522Z","shell.execute_reply.started":"2023-02-01T04:42:49.983477Z","shell.execute_reply":"2023-02-01T04:42:50.078331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.reshape(-1, 28, 28, 1)\nX_test = X_test.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:50.081126Z","iopub.execute_input":"2023-02-01T04:42:50.081511Z","iopub.status.idle":"2023-02-01T04:42:50.087154Z","shell.execute_reply.started":"2023-02-01T04:42:50.081474Z","shell.execute_reply":"2023-02-01T04:42:50.086204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:50.088759Z","iopub.execute_input":"2023-02-01T04:42:50.089494Z","iopub.status.idle":"2023-02-01T04:42:50.101332Z","shell.execute_reply.started":"2023-02-01T04:42:50.089459Z","shell.execute_reply":"2023-02-01T04:42:50.098563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"width_shift_val = 0.1\nheight_shift_val = 0.1\nshear_range_val=10\nzoom_range_val= 0.1\nrandom_rotation_val = 10\ndatagen = tf.keras.preprocessing.image.ImageDataGenerator(rotation_range = random_rotation_val,width_shift_range = width_shift_val,\n                                                          height_shift_range = height_shift_val,zoom_range=zoom_range_val,)\ndatagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:50.104345Z","iopub.execute_input":"2023-02-01T04:42:50.105112Z","iopub.status.idle":"2023-02-01T04:42:50.202496Z","shell.execute_reply.started":"2023-02-01T04:42:50.105034Z","shell.execute_reply":"2023-02-01T04:42:50.201456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stop = EarlyStopping(monitor=\"val_loss\",min_delta=0, patience=12,\n                           verbose=0,mode=\"min\", baseline=None, restore_best_weights=True)\ncheck = ModelCheckpoint(filepath='convnet.keras', save_best_only=True, monitor='val_loss')\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:42:50.204190Z","iopub.execute_input":"2023-02-01T04:42:50.204582Z","iopub.status.idle":"2023-02-01T04:42:50.214007Z","shell.execute_reply.started":"2023-02-01T04:42:50.204543Z","shell.execute_reply":"2023-02-01T04:42:50.211723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'same', \n                 activation ='relu', input_shape = input_shape))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'same', \n                 activation ='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'same', \n                 activation ='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(filters = 128, kernel_size = (3,3),padding = 'same', \n                 activation ='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 128, kernel_size = (3,3),padding = 'same', \n                 activation ='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:44:11.382303Z","iopub.execute_input":"2023-02-01T04:44:11.383156Z","iopub.status.idle":"2023-02-01T04:44:11.552472Z","shell.execute_reply.started":"2023-02-01T04:44:11.383116Z","shell.execute_reply":"2023-02-01T04:44:11.551477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:44:13.191970Z","iopub.execute_input":"2023-02-01T04:44:13.192922Z","iopub.status.idle":"2023-02-01T04:44:13.198130Z","shell.execute_reply.started":"2023-02-01T04:44:13.192879Z","shell.execute_reply":"2023-02-01T04:44:13.197040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=tf.keras.losses.categorical_crossentropy,\n              optimizer=optimizer,\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:44:15.517089Z","iopub.execute_input":"2023-02-01T04:44:15.517883Z","iopub.status.idle":"2023-02-01T04:44:15.528595Z","shell.execute_reply.started":"2023-02-01T04:44:15.517838Z","shell.execute_reply":"2023-02-01T04:44:15.527577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nnum_epoch = 30\nmodel_log = model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),\n                      epochs=num_epoch,\n                      verbose=1,\n                      validation_data=(X_test, y_test),\n                      callbacks = [early_stop, check, learning_rate_reduction])","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:47:45.306739Z","iopub.execute_input":"2023-02-01T04:47:45.307136Z","iopub.status.idle":"2023-02-01T04:54:14.970706Z","shell.execute_reply.started":"2023-02-01T04:47:45.307099Z","shell.execute_reply":"2023-02-01T04:54:14.969792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = np.resize(test_img, (test_img.shape[0], 28, 28, 1))\npred = model.predict(test)\nclasses = pred.argmax(axis=-1)\nclasses","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:43:43.230137Z","iopub.status.idle":"2023-02-01T04:43:43.231777Z","shell.execute_reply.started":"2023-02-01T04:43:43.231587Z","shell.execute_reply":"2023-02-01T04:43:43.231606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsub.Label = classes\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:43:43.234469Z","iopub.status.idle":"2023-02-01T04:43:43.235290Z","shell.execute_reply.started":"2023-02-01T04:43:43.235030Z","shell.execute_reply":"2023-02-01T04:43:43.235054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T04:43:43.236728Z","iopub.status.idle":"2023-02-01T04:43:43.237551Z","shell.execute_reply.started":"2023-02-01T04:43:43.237286Z","shell.execute_reply":"2023-02-01T04:43:43.237310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}