{"cells": [{"source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import keras # for neural networks models\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "dd42215a-d0e7-415a-b4bf-f84a17d98e68", "_uuid": "004de57c7364ed15f1f610fbb3485d77480064df"}, "execution_count": 1}, {"source": ["# Fetch data\n", "data = pd.read_csv('../input/train.csv')\n", "data.head()"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "385a82de-4997-4f0c-aa9c-14f021f0d1f0", "_uuid": "b068c80bd03d833417f774d65ee74f0de9d52de1"}, "execution_count": 2}, {"source": ["# define variables\n", "y = keras.utils.to_categorical(data.label.values, num_classes=10) #One-hot encoding\n", "X = data.drop(['label'], axis=1).values # numpy array where each row is a flatten (vector) image\n", "X = X.reshape((X.shape[0], 28, 28, 1)) # numpy array where each row is a matrix image\n", "X = X/255 # values ranging from 0 (white) to 1 (black)"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "e3f523f0-29a9-4ed7-a27f-90471d07e1ee", "collapsed": true, "_uuid": "82f013236d1cf12ca831111edde0877ce3f44af5"}, "execution_count": 3}, {"source": ["# Split dataset into training set and test set.\n", "from sklearn.model_selection import train_test_split\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "f3fe871d-a572-4e22-b625-7927bc74f2d4", "collapsed": true, "_uuid": "c4d4e7303e7762badd81bbe829e7838c483aebde"}, "execution_count": 28}, {"source": ["# Imports\n", "from keras.models import Sequential\n", "from keras.layers import Dense, Flatten\n", "from keras.layers import Conv2D, MaxPooling2D\n", "from keras.optimizers import Adam"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "1ae1606c-c209-49b6-b0df-730501c38bd2", "collapsed": true, "_uuid": "7ceb4e519b3c0ce6fa862d7a3b5a8ed7ed4e2e8b"}, "execution_count": 5}, {"source": ["# Define convolutional model\n", "def cnn_model():\n", "    # input: 28x28 images with one channel -> (28, 28, 1) tensors.\n", "    model = Sequential()\n", "\n", "    # this applies 32 convolution filters of size 3x3 each.\n", "    model.add(Conv2D(16, (3, 3), padding='same', activation='relu', input_shape=(28, 28, 1)))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", "    model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    \n", "    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", "    model.add(Flatten())\n", "    model.add(Dense(300, activation='relu'))\n", "    model.add(Dense(10, activation='softmax'))\n", "\n", "    adam = Adam(lr=0.001, decay=0.0001)\n", "    model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n", "\n", "    return model"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "16ca9259-56f3-4efa-aa81-6f73649de4c1", "collapsed": true, "_uuid": "26dbe19362614e64dc4a728ec939c14774b7e775"}, "execution_count": 6}, {"source": ["# plot model summary\n", "model = cnn_model()\n", "model.summary()"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "c7f5033a-8299-43b8-9023-3eb1eb884140", "_uuid": "8a57522f0761de1aa1996a911402f51219b785c9"}, "execution_count": 7}, {"source": ["batch_size = 64\n", "epochs = 10\n", "\n", "if (True):\n", "    model = cnn_model()\n", "    model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs)\n", "    loss_and_metric = model.evaluate(X_test, y_test, batch_size=64)\n", "    print('Loss', loss_and_metric[0])\n", "    print('Accuracy:', loss_and_metric[1])"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "b80f63be-ab47-4e60-81a1-c29430f8e415", "_uuid": "c2e773a90c5b68111c29abfb8116c9ff4504a13b"}, "execution_count": 8}, {"source": ["import matplotlib.pyplot as plt\n", "import seaborn as sn\n", "from sklearn.metrics import confusion_matrix\n", "from sklearn.metrics import accuracy_score\n", "from sklearn.metrics import recall_score\n", "\n", "y_true = np.argmax(y_test, axis=1)\n", "y_pred = np.argmax(model.predict(X_test), axis=1)\n", "\n", "# Plot confusion matrix\n", "matrix = confusion_matrix(y_true, y_pred)\n", "sn.heatmap(matrix, annot=True, fmt='d')\n", "\n", "# For seeing where our algorithm is misclassifying numbers\n", "row_sums = matrix.sum(axis=1, keepdims=True)\n", "norm_confusion_matrix = matrix / row_sums\n", "np.fill_diagonal(norm_confusion_matrix, 0)\n", "plt.matshow(norm_confusion_matrix, cmap=plt.cm.gray)\n", "plt.show()"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "f04b61d9-5e32-45ca-9d60-1f450128c286", "_uuid": "f34132e42226872c55f2300f8fce9fe89d8b58aa"}, "execution_count": 27}, {"source": ["# See in which numbers our algorithm is making mistakes.\n", "# for example, our algorithm is missclasifying 6 with 9.\n", "# Code from: https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\n", "\n", "# Errors are difference between predicted labels and true labels\n", "errors = (y_pred - y_true != 0)\n", "\n", "Y_pred_classes_errors = y_pred[errors]\n", "Y_pred_errors = model.predict(X_test)[errors]\n", "Y_true_errors = y_true[errors]\n", "X_val_errors = X_test[errors]\n", "\n", "def display_errors(errors_index,img_errors,pred_errors, obs_errors):\n", "    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n", "    n = 0\n", "    nrows = 2\n", "    ncols = 3\n", "    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n", "    for row in range(nrows):\n", "        for col in range(ncols):\n", "            error = errors_index[n]\n", "            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n", "            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n", "            n += 1\n", "\n", "# Probabilities of the wrong predicted numbers\n", "Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n", "\n", "# Predicted probabilities of the true values in the error set\n", "true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n", "\n", "# Difference between the probability of the predicted label and the true label\n", "delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n", "\n", "# Sorted list of the delta prob errors\n", "sorted_dela_errors = np.argsort(delta_pred_true_errors)\n", "\n", "# Top 6 errors \n", "most_important_errors = sorted_dela_errors[-6:]\n", "\n", "# Show the top 6 errors\n", "display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)\n"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "5c16eb06-6f8b-411c-8857-b97c4e96621f", "_uuid": "4168a02f205754f584cb47d0fd14be97d9a9f5f1"}, "execution_count": 24}, {"source": ["# Training model with data augmentation\n", "# Code from: https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\n", "from keras.preprocessing.image import ImageDataGenerator\n", "idg = ImageDataGenerator(\n", "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n", "        zoom_range = 0.1, # Randomly zoom image \n", "        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n", "        height_shift_range=0.1)  # randomly shift images vertically (fraction of total height)\n", "    \n", "idg.fit(X_train)"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "ebd9fdbf-d406-43f0-abff-95a9024afe90", "_uuid": "42f21d8f9c8658d9e2225d70d0765e90f4123eb8"}, "execution_count": 32}, {"source": ["# Train a new model\n", "batch_size = 64\n", "epochs = 10\n", "new_model = cnn_model()\n", "history = model.fit_generator(idg.flow(X_train, y_train, batch_size=batch_size),\n", "                              epochs=epochs, validation_data=(X_test, y_test))"], "cell_type": "code", "outputs": [], "metadata": {}, "execution_count": 45}, {"source": ["# Plot the loss and accuracy curves for training and validation \n", "fig, ax = plt.subplots(2,1)\n", "ax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\n", "ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\", axes=ax[0])\n", "legend = ax[0].legend(loc='best', shadow=True)\n", "\n", "ax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\n", "ax[1].plot(history.history['val_acc'], color='r', label=\"Validation accuracy\")\n", "legend = ax[1].legend(loc='best', shadow=True)"], "cell_type": "code", "outputs": [], "metadata": {}, "execution_count": 46}, {"source": ["# Read test data\n", "X_test_for_predictions = pd.read_csv('../input/test.csv')\n", "X_test_for_predictions = X_test_for_predictions.values.reshape((X_test_for_predictions.shape[0], 28, 28, 1))\n", "X_test_for_predictions = X_test_for_predictions / 255\n", "predictions = np.argmax(new_model.predict(X_test_for_predictions), axis=1)"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "0133a739-756c-4329-a0d3-bef9cbbce604", "collapsed": true, "_uuid": "5781a4e75c68ad0ad95f6bc1d08227dc6728458e"}, "execution_count": 47}, {"source": ["idx = np.arange(predictions.size).reshape(-1,1) + 1\n", "output = np.hstack((idx, predictions.reshape(-1, 1))).astype(np.int)\n", "np.savetxt('predictions.csv', output, fmt='%i', delimiter=\",\", header='ImageId,Label', comments='')"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "aba28b4b-5802-4476-8689-a35dcb85b460", "collapsed": true, "_uuid": "151c9e56f9fa6f63506c7fa7895585d3c2f2fe68"}, "execution_count": 48}, {"source": [], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "9ee688cf-83ae-4bd3-bfa7-b68bc679839d", "collapsed": true, "_uuid": "0499f3c9370102c2ee1f3614a354948948b4f0b7"}, "execution_count": null}], "metadata": {"kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}, "language_info": {"file_extension": ".py", "name": "python", "version": "3.6.4", "codemirror_mode": {"name": "ipython", "version": 3}, "pygments_lexer": "ipython3", "mimetype": "text/x-python", "nbconvert_exporter": "python"}}, "nbformat": 4, "nbformat_minor": 1}