{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\nfrom glob import glob\nfrom PIL import Image\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport cv2\nimport fnmatch\nimport keras\nfrom time import sleep\nfrom keras.utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Conv2D,MaxPool2D,Dropout,Flatten,BatchNormalization,MaxPooling2D,Activation\nfrom keras.optimizers import RMSprop,Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom keras import backend as k","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"b3c5ce9846b296c0d540c692bcf70f501eef83e0"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ny_train = np.array(df['label'])\nx_train = np.array(df.drop(['label'], axis=1))\nx_train = x_train.reshape(42000, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"87c1b38ab8a80299c51203793c3878b18c31b858"},"cell_type":"code","source":"df = pd.read_csv('../input/test.csv')\nx_check = np.array(df)\nx_check = x_check.reshape(28000, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9bd3f341ceb9cc93ec3eb7644fcd7ea83feff06"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size = 0.2, random_state = 101)\ny_train = to_categorical(y_train, num_classes = 10)\ny_test = to_categorical(y_test, num_classes = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa7437928d3c9215e2a59f60e139eed05d621e5e"},"cell_type":"markdown","source":"import keras\nfrom keras.models import Sequential,Input,Model\nfrom keras.layers import Conv2D, MaxPooling2D, MaxPooling1D, GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, LSTM, TimeDistributed\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.advanced_activations import LeakyReLU \nmodel = Sequential()\nmodel.add(Conv2D(32,(7,7),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2),padding='same'))\nmodel.add(Conv2D(64,(5,5),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2),padding='same'))\nmodel.add(Conv2D(128,(3,3),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2),padding='same'))\nmodel.add(Conv2D(128,(3,3),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2),padding='same'))\nmodel.add(Conv2D(128,(3,3),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2),padding='same'))\nmodel.add(Conv2D(128,(3,3),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2),padding='same'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.15))\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(1000, activation='relu'))\nmodel.add(Dense(10,activation='softmax'))\nmodel.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['accuracy'])\n#model.summary()"},{"metadata":{"trusted":true,"_uuid":"ff58bf2270b9b770e04c57452072c88af84d5528","scrolled":true},"cell_type":"markdown","source":"from keras.callbacks import ModelCheckpoint\nmcp = ModelCheckpoint(filepath='model_check_path.hdf5',monitor=\"val_acc\", save_best_only=True, save_weights_only=False)\nhist = model.fit(x_train,y_train,batch_size = 32, epochs = 100, verbose=1,  validation_split=0.2, callbacks=[mcp])"},{"metadata":{"trusted":true,"_uuid":"4924bd14fc26a2a28fb39e38ddc962596b3cbf3f"},"cell_type":"markdown","source":"fig = plt.figure()\nax = fig.add_subplot(111)\nax.set_facecolor('w')\nax.grid(b=False)\nax.plot(hist.history['acc'], color='red')\nax.plot(hist.history['val_acc'], color ='green')\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='lower right')\nplt.show()"},{"metadata":{"trusted":true,"_uuid":"db7c5514945694865607c301061f22524941f0be"},"cell_type":"markdown","source":"fig = plt.figure()\nax = fig.add_subplot(111)\nax.set_facecolor('w')\nax.grid(b=False)\nax.plot(hist.history['loss'], color='red')\nax.plot(hist.history['val_loss'], color ='green')\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper right')\nplt.show()"},{"metadata":{"trusted":true,"_uuid":"2aff16d3632800e434d733e2c04cccf6632b66fc"},"cell_type":"markdown","source":"model.load_weights('model_check_path.hdf5')"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"5f78858e21789d03d8ca21d1be34ea78d2b36ce0"},"cell_type":"markdown","source":"import keras\nfrom keras.models import Sequential,Input,Model\nfrom keras.layers import Conv2D, MaxPooling2D, MaxPooling1D, GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, LSTM, TimeDistributed\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.advanced_activations import LeakyReLU\nmodel = Sequential()\nmodel.add(Conv2D(32,(7,7),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.15))\nmodel.add(Conv2D(64,(5,5),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.15))\nmodel.add(Conv2D(128,(3,3),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.15))\nmodel.add(Conv2D(128,(3,3),activation='relu',padding='same'))\nmodel.add(MaxPooling2D((2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.15))\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(1000, activation='relu'))\nmodel.add(Dense(10,activation='softmax'))\nmodel.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['accuracy'])\n#model.summary()\n"},{"metadata":{"trusted":true,"_uuid":"5f218ceb50790f596dccdca3070a1e05a4d2050a","scrolled":false},"cell_type":"markdown","source":"from keras.callbacks import ModelCheckpoint\nmcp = ModelCheckpoint(filepath='model_check_path_2.hdf5',monitor=\"val_acc\", save_best_only=True, save_weights_only=False)\nhist = model.fit(x_train,y_train,batch_size = 32, epochs = 50, verbose=1,  validation_split=0.2, callbacks=[mcp])"},{"metadata":{"trusted":true,"_uuid":"3aa3ee6c75b12193309e2e0ce6decc6903eb447c"},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential,Input,Model\nfrom keras.layers import Conv2D, MaxPooling2D, MaxPooling1D, GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, LSTM, TimeDistributed\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.applications.vgg16 import VGG16\ndef base_model(x):\n    BATCH_NORM = x\n    model = Sequential()\n\n    model.add(Conv2D(64, (3, 3), padding='same', input_shape=x_train.shape[1:], name='block1_conv1'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(64, (3, 3), padding='same', name='block1_conv2'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool'))\n\n    model.add(Conv2D(128, (3, 3), padding='same', name='block2_conv1'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(128, (3, 3), padding='same', name='block2_conv2'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool'))\n\n    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv1'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv2'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv3'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(256, (3, 3), padding='same', name='block3_conv4'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv1'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv2'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv3'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block4_conv4'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv1'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv2'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv3'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Conv2D(512, (3, 3), padding='same', name='block5_conv4'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n\n    model.add(Flatten())\n\n    model.add(Dense(4096))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n    model.add(Dropout(0.5))\n\n    model.add(Dense(4096, name='fc2'))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('relu'))\n    model.add(Dropout(0.5))\n\n    model.add(Dense(10))\n    model.add(BatchNormalization()) if BATCH_NORM else None\n    model.add(Activation('softmax'))\n\n    model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['accuracy'])\n    return model\n#model.summary()\ncnn = base_model(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d7edfa1419f96750506d8721254178eeb981578"},"cell_type":"markdown","source":"from keras.callbacks import ModelCheckpoint\nmcp = ModelCheckpoint(filepath='model_check_path_2.hdf5',monitor=\"val_acc\", save_best_only=True, save_weights_only=False)\nhist = cnn.fit(x_train,y_train,batch_size = 32, epochs = 60, verbose=1,  validation_split=0.2, callbacks=[mcp])"},{"metadata":{"trusted":true,"_uuid":"82d46ceef674bc90b1ef6aea28c6690b1174995a"},"cell_type":"code","source":"cnn.load_weights('model_check_path_2.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0afa6e7166ffe4fbef8dec67e4eec504aef4fd10"},"cell_type":"code","source":"from sklearn.metrics import classification_report\npred = cnn.predict(x_test)\nprint(classification_report(np.argmax(y_test, axis = 1),np.argmax(pred, axis = 1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d305a882a2422ce0858d12123d4bd203f9b6d2d"},"cell_type":"markdown","source":"def plot_confusion_matrix(cm,\n                          target_names,\n                          title='Confusion matrix',\n                          cmap=None,\n                          normalize=True):\n    import matplotlib.pyplot as plt\n    import numpy as np\n    import itertools\n\n    accuracy = np.trace(cm) / float(np.sum(cm))\n    misclass = 1 - accuracy\n\n    if cmap is None:\n        cmap = plt.get_cmap('Blues')\n\n    plt.figure(figsize=(8, 6))\n    plt.grid(b=False)\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n\n    if target_names is not None:\n        tick_marks = np.arange(len(target_names))\n        plt.xticks(tick_marks, target_names, rotation=45)\n        plt.yticks(tick_marks, target_names)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n\n    thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        if normalize:\n            plt.text(j, i, \"{:0.4f}\".format(cm[i, j]),\n                     horizontalalignment=\"center\",\n                     color=\"white\" if cm[i, j] > thresh else \"black\")\n        else:\n            plt.text(j, i, \"{:,}\".format(cm[i, j]),\n                     horizontalalignment=\"center\",\n                     color=\"white\" if cm[i, j] > thresh else \"black\")\n\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label\\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))\n    plt.show()"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"a17517ddef46fc4ba0840c3db9472cc84a629937"},"cell_type":"markdown","source":"\nfrom sklearn.metrics import confusion_matrix\ncm = confusion_matrix(np.argmax(y_test, axis = 1),np.argmax(pred, axis = 1))\nplot_confusion_matrix(cm = cm,\n                      normalize    = False,\n                      cmap ='Reds',\n                      target_names = ['0','1', '2','3', '4', '5', '6', '7', '8', '9'],\n                      title        = \"Confusion Matrix\")"},{"metadata":{"trusted":true,"_uuid":"133015da70d99f6e56f09a36363cf12689f5f474"},"cell_type":"code","source":"prediction = cnn.predict(x_check)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"410c6f8ae3f449b6472f0b64fa0a42f77002d742"},"cell_type":"markdown","source":"for i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.imshow(x_check[i].reshape(28,28), cmap='gray', interpolation='none')\n    plt.title(\"{}\".format(prediction[i].argmax()))"},{"metadata":{"trusted":true,"_uuid":"073c9e644f1e0dc022ca626d1743bc09bacbcef9"},"cell_type":"code","source":"pred_sub = []\nfor i in range(len(prediction)):\n    pred_sub.append(prediction[i].argmax())\npred_sub = np.array(pred_sub)\npred_sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f490edcf18f420d015ac9332befcabed66cb3b81"},"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['ImageId']=np.array([i+1 for i in range(len(pred_sub))])\nsubmission['Label']=pred_sub\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f9201102991cb5079ee5fd154dd92b0741f035e"},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}