{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential\nfrom keras.utils import to_categorical\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras import optimizers\nimport tensorflow as tf\n# tf.logging.set_verbosity(tf.logging.ERROR)\n\nimport itertools\nimport pandas as pd\nimport seaborn as sns\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Dense\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.convolutional import Conv2D # to add convolutional layers\nfrom keras.layers.convolutional import MaxPooling2D # to add pooling layers\nfrom keras.layers import Flatten # to flatten data for fully connected layers\nfrom keras.layers import Dropout # to avoid overfitting\nfrom keras.layers import Activation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_loss_accuracy(history):\n    historydf = pd.DataFrame(history.history, index=history.epoch)\n    plt.figure(figsize=(8, 6))\n    historydf.plot(ylim=(0, max(1, historydf.values.max())))\n    Val_loss = history.history['val_loss'][-1]\n    Val_acc = history.history['val_accuracy'][-1]\n    Train_loss = history.history['loss'][-1]\n    Train_acc = history.history['accuracy'][-1]\n    plt.title('Validation Loss: %.3f, Validation Accuracy: %.3f, Training Loss: %.3f, Training Accuracy: %.3f' % (Val_loss, Val_acc, Train_loss, Train_acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = train_df.columns\nX = train_df[columns[columns != 'label']]\ny = train_df['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, random_state = 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.values.reshape(X_train.shape[0], 28, 28, 1).astype('float32')\nX_test = X_test.values.reshape(X_test.shape[0], 28, 28, 1).astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train / 255 # normalize training data\nX_test = X_test / 255 # normalize test data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = to_categorical(y_train)\ny_test = to_categorical(y_test)\n\nnum_classes = y_test.shape[1] # number of categories","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convolutional_model():\n    ADAMAX = optimizers.Adamax(lr = 0.002, beta_1 = 0.9, beta_2 = 0.999)\n    # create model\n    model = Sequential()\n    model.add(Conv2D(32, (4, 4), activation = 'relu', input_shape = (28, 28, 1)))\n    model.add(Conv2D(64, (3, 3), activation = 'relu'))\n    model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n    model.add(BatchNormalization(axis = -1))\n    model.add(Dropout(0.2))\n    \n    model.add(Conv2D(128, (3, 3), activation = 'relu'))\n    model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n    model.add(BatchNormalization(axis = -1))\n    model.add(Dropout(0.2))\n    \n    model.add(Conv2D(128, (2, 2), activation = 'relu'))\n    model.add(Conv2D(256, (2, 2), activation = 'relu'))\n    model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n    model.add(BatchNormalization(axis = -1))\n    model.add(Dropout(0.2))\n    \n    model.add(Flatten())\n    model.add(Dense(256, activation = 'relu'))\n    model.add(BatchNormalization())\n    model.add(Dense(128, activation = 'relu'))\n    model.add(Activation('relu'))\n    model.add(Dropout(0.2))\n    model.add(Dense(num_classes, activation = 'softmax'))\n    \n    # Compile model\n    model.compile(optimizer = ADAMAX, loss = 'categorical_crossentropy',  metrics = ['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gen = ImageDataGenerator(rotation_range = 12, width_shift_range = 0.1, shear_range = 0.1,\n                         height_shift_range = 0.1, zoom_range = 0.1, fill_mode = 'nearest', horizontal_flip = False,\n                         vertical_flip = False, featurewise_center = False,\n                         samplewise_center = False, featurewise_std_normalization = False,\n                         samplewise_std_normalization = False)\ntest_gen = ImageDataGenerator()\n\n# Create batches to  train models faster\ntrain_generator = gen.flow(X_train, y_train, batch_size = 32)\ntest_generator = test_gen.flow(X_test, y_test, batch_size = 32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use annelar to gradually decrese the learning rate to improve generalization\n\nreduce_lr = ReduceLROnPlateau(monitor = 'val_accuracy', patience = 3, verbose = 1, factor = 0.4, min_lr = 0.00002,\n                                            mode = 'auto', cooldown = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# build the model\nmodel = convolutional_model()\nepochs = 80\n# fit the model\n\n#history = model.fit(X_train, y_train, validation_data = (X_test, y_test), epochs = 30, batch_size = 200, verbose = 2)\n\nhistory = model.fit_generator(train_generator, steps_per_epoch = 40000//16, epochs = epochs, \n                              validation_data = test_generator, validation_steps = 10000//8, verbose = 1,\n                              callbacks=[reduce_lr])\n\n# evaluate the model\nscores = model.evaluate(X_test, y_test, verbose = 0)\nprint(\"Accuracy: {} \\n Error: {}\".format(scores[1], 100-scores[1]*100))\nplot_loss_accuracy(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_loss_accuracy(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = np.argmax(model.predict(X_test), axis = 1)\nY_true = np.argmax(y_test, axis = 1)\ncm = confusion_matrix(Y_true, y_pred)\nplot_confusion_matrix(cm, classes = range(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = test_df.values.reshape(test_df.shape[0], 28, 28, 1).astype('float32')\ntest_data = test_data / 255\nY_pred = model.predict(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_pred = np.argmax(Y_pred,axis = 1)\nY_pred = pd.Series(Y_pred, name = \"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({\n                  \"ImageId\": pd.Series(range(1, len(Y_pred)+1)),\n                  \"Label\": pd.Series(Y_pred)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('/kaggle/working/Submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}