{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'svg'\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport itertools\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, BatchNormalization, Dropout, Flatten, Lambda, Conv2D, MaxPooling2D\nfrom tensorflow.keras.utils import plot_model, to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, confusion_matrix\n\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\n\ny_train = train['label']\nx_train = train.drop(['label'], axis=1)\n\ny_train = to_categorical(y_train)\n\nx_train = x_train.astype('float32') / 255.0\ntest = test.astype('float32') / 255.0\n\nx_train = x_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)\n\nprint(x_train.shape)\nprint(test.shape)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":14.361345,"end_time":"2022-05-05T18:05:59.313494","exception":false,"start_time":"2022-05-05T18:05:44.952149","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:04:40.657886Z","iopub.execute_input":"2022-05-12T23:04:40.658556Z","iopub.status.idle":"2022-05-12T23:04:44.925752Z","shell.execute_reply.started":"2022-05-12T23:04:40.658513Z","shell.execute_reply":"2022-05-12T23:04:44.924747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"papermill":{"duration":0.037903,"end_time":"2022-05-05T18:05:59.375083","exception":false,"start_time":"2022-05-05T18:05:59.33718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:04:44.927663Z","iopub.execute_input":"2022-05-12T23:04:44.928078Z","iopub.status.idle":"2022-05-12T23:04:44.938878Z","shell.execute_reply.started":"2022-05-12T23:04:44.928019Z","shell.execute_reply":"2022-05-12T23:04:44.938037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(facecolor='white',figsize=(10,20))\nplt.subplot(131)\nplt.imshow(x_train[338,:,:,0], cmap=plt.cm.binary)\nplt.subplot(132)\nplt.imshow(x_train[342,:,:,0], cmap=plt.cm.binary)\nplt.subplot(133)\nplt.imshow(x_train[125,:,:,0], cmap=plt.cm.binary)\nplt.title(' ')","metadata":{"papermill":{"duration":0.52444,"end_time":"2022-05-05T18:06:00.030639","exception":false,"start_time":"2022-05-05T18:05:59.506199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:04:44.940801Z","iopub.execute_input":"2022-05-12T23:04:44.941084Z","iopub.status.idle":"2022-05-12T23:04:45.329602Z","shell.execute_reply.started":"2022-05-12T23:04:44.941035Z","shell.execute_reply":"2022-05-12T23:04:45.328944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel = Sequential()\nmodel.add(Conv2D(32,(3,3), padding='same', activation='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(32,(3,3), padding='same', activation='relu', input_shape = (28,28,1)))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Dropout(0.1))\n\nmodel.add(Conv2D(64,(3,3), padding='same', activation='relu'))\nmodel.add(Conv2D(64,(3,3), padding='same', activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv2D(128,(3,3), padding='same', activation='relu'))\nmodel.add(Conv2D(64,(3,3), padding='same', activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Dropout(0.3))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dense(10, activation='softmax'))\nmodel.summary()","metadata":{"papermill":{"duration":2.944389,"end_time":"2022-05-05T18:06:03.001768","exception":false,"start_time":"2022-05-05T18:06:00.057379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:04:45.331927Z","iopub.execute_input":"2022-05-12T23:04:45.332422Z","iopub.status.idle":"2022-05-12T23:04:45.45394Z","shell.execute_reply.started":"2022-05-12T23:04:45.332385Z","shell.execute_reply":"2022-05-12T23:04:45.45319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model, show_shapes=True, show_layer_names=True)","metadata":{"papermill":{"duration":0.958379,"end_time":"2022-05-05T18:06:04.055158","exception":false,"start_time":"2022-05-05T18:06:03.096779","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:04:45.498775Z","iopub.execute_input":"2022-05-12T23:04:45.499089Z","iopub.status.idle":"2022-05-12T23:04:45.779595Z","shell.execute_reply.started":"2022-05-12T23:04:45.499044Z","shell.execute_reply":"2022-05-12T23:04:45.778741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\nBATCH_SIZE = 64\nsteps = x_train.shape[0] // BATCH_SIZE\nhistory = model.fit(x_train, y_train,\n              epochs= 40,\n              steps_per_epoch=steps,\n              batch_size = BATCH_SIZE,\n              validation_split=0.2,\n              shuffle=True)","metadata":{"papermill":{"duration":30.964088,"end_time":"2022-05-05T18:06:35.048269","exception":false,"start_time":"2022-05-05T18:06:04.084181","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:04:45.781417Z","iopub.execute_input":"2022-05-12T23:04:45.78181Z","iopub.status.idle":"2022-05-12T23:07:08.676977Z","shell.execute_reply.started":"2022-05-12T23:04:45.781775Z","shell.execute_reply":"2022-05-12T23:07:08.676054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.facecolor'] = 'white'\nhistory_dict = history.history\nacc = history_dict['accuracy']\nval_acc = history_dict['val_accuracy']\nepochs = range(1, len(acc) + 1)\nplt.plot(epochs, acc, 'bo', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('Training Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"papermill":{"duration":0.425186,"end_time":"2022-05-05T18:06:35.656753","exception":false,"start_time":"2022-05-05T18:06:35.231567","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:08.680777Z","iopub.execute_input":"2022-05-12T23:07:08.681034Z","iopub.status.idle":"2022-05-12T23:07:08.88358Z","shell.execute_reply.started":"2022-05-12T23:07:08.681001Z","shell.execute_reply":"2022-05-12T23:07:08.882932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = model.predict(test)\n# select the index with the maximum probability\nresults = np.argmax(test_predictions,axis = 1)\nsubmission = pd.DataFrame({'ImageId': range(1,28001), 'Label': results})\nsubmission.head(10)\nsubmission.to_csv(\"submission.csv\",index=False)","metadata":{"papermill":{"duration":3.143227,"end_time":"2022-05-05T18:06:38.98638","exception":false,"start_time":"2022-05-05T18:06:35.843153","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:08.885253Z","iopub.execute_input":"2022-05-12T23:07:08.885703Z","iopub.status.idle":"2022-05-12T23:07:10.587449Z","shell.execute_reply.started":"2022-05-12T23:07:08.885666Z","shell.execute_reply":"2022-05-12T23:07:10.586593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=2017)","metadata":{"papermill":{"duration":0.666313,"end_time":"2022-05-05T18:06:39.838296","exception":false,"start_time":"2022-05-05T18:06:39.171983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:10.58879Z","iopub.execute_input":"2022-05-12T23:07:10.589037Z","iopub.status.idle":"2022-05-12T23:07:10.973896Z","shell.execute_reply.started":"2022-05-12T23:07:10.589004Z","shell.execute_reply":"2022-05-12T23:07:10.973175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.6, \n                                            min_lr=0.00001)","metadata":{"papermill":{"duration":0.286453,"end_time":"2022-05-05T18:06:40.457569","exception":false,"start_time":"2022-05-05T18:06:40.171116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:10.975162Z","iopub.execute_input":"2022-05-12T23:07:10.975424Z","iopub.status.idle":"2022-05-12T23:07:10.980319Z","shell.execute_reply.started":"2022-05-12T23:07:10.975391Z","shell.execute_reply":"2022-05-12T23:07:10.979624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 256\nhistory = model.fit(\n            x_train, y_train,\n            validation_data = (x_val, y_val),\n            batch_size = BATCH_SIZE,\n            steps_per_epoch = x_train.shape[0] // BATCH_SIZE,\n            epochs = 15,\n            verbose = 1,\n            callbacks=[learning_rate_reduction]\n            )","metadata":{"papermill":{"duration":41.41724,"end_time":"2022-05-05T18:07:22.059998","exception":false,"start_time":"2022-05-05T18:06:40.642758","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:10.981669Z","iopub.execute_input":"2022-05-12T23:07:10.982121Z","iopub.status.idle":"2022-05-12T23:07:52.568616Z","shell.execute_reply.started":"2022-05-12T23:07:10.982056Z","shell.execute_reply":"2022-05-12T23:07:52.567812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nplt.figure()\nplt.plot(acc,color = 'green',label = 'Training Acuracy')\nplt.plot(val_acc,color = 'red',label = 'Validation Accuracy')\nplt.legend()","metadata":{"papermill":{"duration":0.640794,"end_time":"2022-05-05T18:07:23.035158","exception":false,"start_time":"2022-05-05T18:07:22.394364","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:52.570184Z","iopub.execute_input":"2022-05-12T23:07:52.570431Z","iopub.status.idle":"2022-05-12T23:07:52.809773Z","shell.execute_reply.started":"2022-05-12T23:07:52.570399Z","shell.execute_reply":"2022-05-12T23:07:52.808993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix',cmap=plt.cm.Reds):\n    '''This function prints and plots the confusion matrix.\n       Normalization can be applied by setting `normalize=True`.\n    '''\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"papermill":{"duration":0.435507,"end_time":"2022-05-05T18:07:23.811151","exception":false,"start_time":"2022-05-05T18:07:23.375644","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:52.811253Z","iopub.execute_input":"2022-05-12T23:07:52.812348Z","iopub.status.idle":"2022-05-12T23:07:52.824291Z","shell.execute_reply.started":"2022-05-12T23:07:52.812303Z","shell.execute_reply":"2022-05-12T23:07:52.823489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val_pred = model.predict(x_val)\ny_val_pred = np.argmax(y_val_pred,axis = 1)\ny_val1 = np.argmax(y_val,axis = 1)\n#val_preds_class = np.where(val_preds.ravel()>0.5,1,0) \nprint('Out of Fold Accuracy is {:.5}'.format(accuracy_score(y_val1, y_val_pred)))","metadata":{"papermill":{"duration":1.116247,"end_time":"2022-05-05T18:07:25.485304","exception":false,"start_time":"2022-05-05T18:07:24.369057","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:52.825754Z","iopub.execute_input":"2022-05-12T23:07:52.826027Z","iopub.status.idle":"2022-05-12T23:07:53.601182Z","shell.execute_reply.started":"2022-05-12T23:07:52.825987Z","shell.execute_reply":"2022-05-12T23:07:53.600356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.facecolor'] = 'white'\nclass_names = [i for i in range(1,11)]\nplot_confusion_matrix(confusion_matrix(y_val1, y_val_pred),\n                      classes=class_names, \n                      title='MINST confusion_matrix')","metadata":{"papermill":{"duration":1.421092,"end_time":"2022-05-05T18:07:27.24634","exception":false,"start_time":"2022-05-05T18:07:25.825248","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-12T23:07:53.602412Z","iopub.execute_input":"2022-05-12T23:07:53.602937Z","iopub.status.idle":"2022-05-12T23:07:54.282823Z","shell.execute_reply.started":"2022-05-12T23:07:53.602897Z","shell.execute_reply":"2022-05-12T23:07:54.281983Z"},"trusted":true},"execution_count":null,"outputs":[]}]}