{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt \nfrom collections import Counter\nfrom sklearn.metrics import confusion_matrix\nimport itertools\nimport seaborn as sns\n%matplotlib inline","metadata":{"_cell_guid":"2d75481f-2b19-d824-6a77-a6f8fa1f63f8","_execution_state":"idle","_uuid":"e8e286a4d699f3294f0ff773baba59f015ae11b9","_active":false,"execution":{"iopub.status.busy":"2021-09-27T13:50:58.781763Z","iopub.execute_input":"2021-09-27T13:50:58.782779Z","iopub.status.idle":"2021-09-27T13:50:58.796065Z","shell.execute_reply.started":"2021-09-27T13:50:58.782219Z","shell.execute_reply":"2021-09-27T13:50:58.794869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading The Dataset","metadata":{"_cell_guid":"f9f804d7-72e7-415f-88b9-f43ded389d39","_uuid":"4375276c8b31c78a81859766e49db726e73cf4b5"}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest= pd.read_csv(\"../input/test.csv\")","metadata":{"_cell_guid":"d2ecfe40-ec05-6623-6f2d-a296d7eeec9e","_execution_state":"idle","_uuid":"1de17f87d4eb39ebb9772e29bbc70186d09ac927","_active":true,"execution":{"iopub.status.busy":"2021-09-27T13:50:58.800817Z","iopub.execute_input":"2021-09-27T13:50:58.801213Z","iopub.status.idle":"2021-09-27T13:51:05.974911Z","shell.execute_reply.started":"2021-09-27T13:50:58.801157Z","shell.execute_reply":"2021-09-27T13:51:05.974119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.head()\n# train.shape\n# test.head()\n# test.shape\n\ntrain['label'].value_counts()","metadata":{"_cell_guid":"74d1193c-49c0-4f83-bb76-9280bf544959","_uuid":"8da998a4c8f1e8ee218a6118217516754fb51868","execution":{"iopub.status.busy":"2021-09-27T13:51:05.976937Z","iopub.execute_input":"2021-09-27T13:51:05.977477Z","iopub.status.idle":"2021-09-27T13:51:06.000637Z","shell.execute_reply.started":"2021-09-27T13:51:05.97725Z","shell.execute_reply":"2021-09-27T13:51:05.995841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = (train.iloc[:,1:].values).astype('float32')\ny_train = train.iloc[:,0].values.astype('int32') \nx_test = test.values.astype('float32')","metadata":{"_cell_guid":"04b01849-64c8-4701-9d1a-4622f827b1a2","_uuid":"9d47ae3027e049f31a0a3046da9db13c72c82e60","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.002912Z","iopub.execute_input":"2021-09-27T13:51:06.003321Z","iopub.status.idle":"2021-09-27T13:51:06.20732Z","shell.execute_reply.started":"2021-09-27T13:51:06.003255Z","shell.execute_reply":"2021-09-27T13:51:06.206031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train/255.0\nx_test = x_test/255.0","metadata":{"_cell_guid":"2af344a9-8fbe-4b2f-af5e-a108adf04722","_uuid":"29756ff1b015740f63a1e3b0ee707d4ee3ee0aab","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.215579Z","iopub.execute_input":"2021-09-27T13:51:06.218363Z","iopub.status.idle":"2021-09-27T13:51:06.433644Z","shell.execute_reply.started":"2021-09-27T13:51:06.218298Z","shell.execute_reply":"2021-09-27T13:51:06.432258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(y_train.shape)\nprint(x_test.shape)","metadata":{"_cell_guid":"72e903bd-ad8d-4751-ba2d-0deddadd4ab3","_uuid":"ac95a6317ce1baebb4f17d18388028318f48b509","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.440078Z","iopub.execute_input":"2021-09-27T13:51:06.443077Z","iopub.status.idle":"2021-09-27T13:51:06.455373Z","shell.execute_reply.started":"2021-09-27T13:51:06.443014Z","shell.execute_reply":"2021-09-27T13:51:06.45411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = x_train.reshape(x_train.shape[0], 28, 28,1)\nX_test = x_test.reshape(x_test.shape[0], 28, 28,1)","metadata":{"_cell_guid":"5d6167d5-1e96-40e6-af0c-b960567ef033","_uuid":"5e4de6b35dbd4db773e890c4d16840485a4f74ad","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.459587Z","iopub.execute_input":"2021-09-27T13:51:06.461356Z","iopub.status.idle":"2021-09-27T13:51:06.479449Z","shell.execute_reply.started":"2021-09-27T13:51:06.461283Z","shell.execute_reply":"2021-09-27T13:51:06.478147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T13:51:06.48539Z","iopub.execute_input":"2021-09-27T13:51:06.488182Z","iopub.status.idle":"2021-09-27T13:51:06.499395Z","shell.execute_reply.started":"2021-09-27T13:51:06.488113Z","shell.execute_reply":"2021-09-27T13:51:06.498283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nbatch_size = 64\nclasses = 10\nepochs = 15\ninput_shape = (28, 28, 1)","metadata":{"_cell_guid":"bb79bc10-9ba2-44fe-a349-570520cd8a80","_uuid":"297c1b8aab418daee490ae79f474ca7814d86378","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.505125Z","iopub.execute_input":"2021-09-27T13:51:06.505871Z","iopub.status.idle":"2021-09-27T13:51:06.519863Z","shell.execute_reply.started":"2021-09-27T13:51:06.505812Z","shell.execute_reply":"2021-09-27T13:51:06.518516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = keras.utils.to_categorical(y_train, classes)\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=42)","metadata":{"_cell_guid":"8c4ecbcb-268c-48e3-ab93-9015e12d0e79","_uuid":"7a5463286287b10f7bdbbbfae8fabf764f6d5b1d","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.524987Z","iopub.execute_input":"2021-09-27T13:51:06.526945Z","iopub.status.idle":"2021-09-27T13:51:06.954933Z","shell.execute_reply.started":"2021-09-27T13:51:06.526712Z","shell.execute_reply":"2021-09-27T13:51:06.954097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n# tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n# with tpu_strategy.scope():\n\n    model = Sequential()\n    model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal',input_shape=input_shape))\n    model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal'))\n    model.add(MaxPool2D((2, 2)))\n    model.add(Dropout(0.20))\n    model.add(Conv2D(64, (3, 3), activation='relu',padding='same',kernel_initializer='he_normal'))\n    model.add(Conv2D(64, (3, 3), activation='relu',padding='same',kernel_initializer='he_normal'))\n    model.add(MaxPool2D(pool_size=(2, 2)))\n    model.add(Dropout(0.25))\n    model.add(Conv2D(128, (3, 3), activation='relu',padding='same',kernel_initializer='he_normal'))\n    model.add(Dropout(0.25))\n    model.add(Flatten())\n    model.add(Dense(128, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.25))\n    model.add(Dense(classes, activation='softmax'))\n\n    model.compile(loss=keras.losses.categorical_crossentropy,\n              optimizer=keras.optimizers.RMSprop(),\n              metrics=['accuracy'])\n\n    callback = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.3, \n                                            min_lr=0.0001)\n\n    datagen = ImageDataGenerator(\n        rotation_range=15,\n        zoom_range = 0.1, \n        width_shift_range=0.1,\n        height_shift_range=0.1) ","metadata":{"_cell_guid":"097867df-c77b-4ca8-988a-7baea6f4e387","_uuid":"577c7efb2d90f284383906e712a68df6233fe4e0","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:06.956544Z","iopub.execute_input":"2021-09-27T13:51:06.956871Z","iopub.status.idle":"2021-09-27T13:51:07.458079Z","shell.execute_reply.started":"2021-09-27T13:51:06.956816Z","shell.execute_reply":"2021-09-27T13:51:07.456668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"_cell_guid":"8dd6be49-de4f-4e76-a6c6-5f2dc9d4f304","_uuid":"9197045f0084aa72f0bd8ece8fb03eca9ab08622","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:07.459981Z","iopub.execute_input":"2021-09-27T13:51:07.46065Z","iopub.status.idle":"2021-09-27T13:51:07.481106Z","shell.execute_reply.started":"2021-09-27T13:51:07.460591Z","shell.execute_reply":"2021-09-27T13:51:07.480065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen.fit(X_train)\nh = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 1, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[callback],)","metadata":{"_cell_guid":"33d2485e-df3b-415b-bf2c-3cc1d1afddc1","_uuid":"fc13b3bb1a906117ae78cfd03639e165072f2e45","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:51:07.482485Z","iopub.execute_input":"2021-09-27T13:51:07.48308Z","iopub.status.idle":"2021-09-27T13:55:47.243101Z","shell.execute_reply.started":"2021-09-27T13:51:07.483025Z","shell.execute_reply":"2021-09-27T13:55:47.241007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic Simple Plot And Evaluation","metadata":{"_cell_guid":"e0f319f7-c701-4ce4-b100-a0db07b9edef","_uuid":"4410fb664affc57163466e4949390b9b97a5d291"}},{"cell_type":"code","source":"final_loss, final_acc = model.evaluate(X_val, Y_val, verbose=0)\nprint(final_loss)\nprint(final_acc)","metadata":{"_cell_guid":"19003034-e42b-4ec4-99d6-abefc7411451","_uuid":"0963a61adf30eba3b8c0d885eee69063e086634d","execution":{"iopub.status.busy":"2021-09-27T13:55:47.245902Z","iopub.execute_input":"2021-09-27T13:55:47.246219Z","iopub.status.idle":"2021-09-27T13:55:47.583104Z","shell.execute_reply.started":"2021-09-27T13:55:47.246161Z","shell.execute_reply":"2021-09-27T13:55:47.582224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This code is taken straight from the SKLEARN website, an nice way of viewing confusion matrix.\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n","metadata":{"_cell_guid":"2c64c843-e1d0-449f-b5a5-e51955a30caf","_uuid":"f5355803304624fae4bde1b4829d6329433395d6","execution":{"iopub.status.busy":"2021-09-27T13:55:47.586572Z","iopub.execute_input":"2021-09-27T13:55:47.586854Z","iopub.status.idle":"2021-09-27T13:55:47.598095Z","shell.execute_reply.started":"2021-09-27T13:55:47.586801Z","shell.execute_reply":"2021-09-27T13:55:47.596944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = model.predict(X_val)\nY_pred_classes = np.argmax(Y_pred, axis = 1) \nY_true = np.argmax(Y_val, axis = 1) \nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \nplot_confusion_matrix(confusion_mtx, classes = range(10))","metadata":{"_cell_guid":"04c6a010-c928-4acb-bfed-3426a474cf18","_uuid":"cfffb6b672be442d54317e6d92b2c0b180a91934","_execution_state":"idle","execution":{"iopub.status.busy":"2021-09-27T13:55:47.599819Z","iopub.execute_input":"2021-09-27T13:55:47.600499Z","iopub.status.idle":"2021-09-27T13:55:49.108379Z","shell.execute_reply.started":"2021-09-27T13:55:47.600438Z","shell.execute_reply":"2021-09-27T13:55:49.107296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_classes = model.predict_classes(X_test)\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predicted_classes)+1)),\n                         \"Label\": predicted_classes})\nsubmissions.to_csv(\"vb.csv\", index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T13:55:49.110216Z","iopub.execute_input":"2021-09-27T13:55:49.110835Z","iopub.status.idle":"2021-09-27T13:55:50.993167Z","shell.execute_reply.started":"2021-09-27T13:55:49.110734Z","shell.execute_reply":"2021-09-27T13:55:50.992295Z"},"trusted":true},"execution_count":null,"outputs":[]}]}