{"nbformat_minor": 1, "nbformat": 4, "cells": [{"source": "Table of Contents -\n1. [Loading libraries and data](#load)\n* [Displaying images](#display)\n* [Preparing data for modeling](#prep)\n* [Multi-layer Perceptron using scikit-learn](#sklearn-mlp)\n* [Multi-layer Perceptron using Keras](#keras)", "outputs": [], "cell_type": "markdown", "metadata": {"_uuid": "60f79c40685939958184aa46084a9d47dfc2e0cb", "_cell_guid": "91bacfc9-49b2-4f8e-be1d-8e7adf78844f"}, "execution_count": null}, {"source": "## Loading libraries and data <a name=\"load\"></a>", "outputs": [], "cell_type": "markdown", "metadata": {"_uuid": "a85456fe2cc155871e32774658bde6517fedc80f", "_cell_guid": "a696ad14-8d34-41ab-9d25-f771ce65a4ed"}, "execution_count": null}, {"source": "# Data analysis\nimport pandas as pd\nimport numpy as np\nfrom scipy import stats, integrate\n\n# Scikit-learn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.preprocessing import OneHotEncoder\n\n# Keras\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.utils import np_utils\n\n# Data visualization\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\n# List of files available in the input directory.\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "2adcdf9380688008ca6ccde3c58c75117e86bca4", "_cell_guid": "a5b97c63-0535-49db-8906-68c516404b90"}, "execution_count": 1}, {"source": "# The training set is quite large (75 MB), we read only the first 10 records for now\ntrain = pd.read_csv('../input/train.csv', nrows=10)", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "c1d79f42d34156de904f1d4d259915878e8b1b70", "_cell_guid": "6b42f367-6381-4fd5-aa19-e2fa5927090b"}, "execution_count": 2}, {"source": "## Displaying images <a name=\"display\"></a>", "outputs": [], "cell_type": "markdown", "metadata": {"_uuid": "35da7c1d5713d1c2da76d29341351e47618c4f2f", "_cell_guid": "1ed20d27-6da5-42e4-bf00-b4b7d1d7f748"}, "execution_count": null}, {"source": "train", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "e17c033f72d1c1fd688a031682d01f0163c605a8", "_cell_guid": "c86779d9-1c1e-42dc-9edc-6f9032c009a5"}, "execution_count": 3}, {"source": "# Display the first 'n' images of a data set\nf = plt.figure(figsize=(10,4))\n# Loop over the images of the training set\nfor i in range(10):\n    ax = plt.subplot(2, 5, i+1)\n    ax.axis('off')\n    image = train.iloc[i, 1:].values.reshape(28, 28)\n    ax.set_title(train['label'].iloc[i])\n    imgplot = plt.imshow(image, cmap='binary')\n\nplt.show()", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "ecd42b3cdcc6022ae52789699be35d264b3469c0", "_cell_guid": "23252cb9-756f-4dc2-8a3d-d935fda08b01"}, "execution_count": 4}, {"source": "", "outputs": [], "cell_type": "markdown", "metadata": {"collapsed": true, "trusted": false, "_uuid": "76b5dac364bf7d98a759b512e7abb5c3ab5128d0", "_cell_guid": "8bffdd61-ab41-4464-8452-ed90ffac6208"}, "execution_count": null}, {"source": "## Preparing data for modeling <a name=\"prep\"></a>", "outputs": [], "cell_type": "markdown", "metadata": {"_uuid": "97a658b6bfe9a9c3d430c00aafe56e3d79622389", "_cell_guid": "81fa89de-66e1-4187-8d52-c1774f64686e"}, "execution_count": null}, {"source": "# Load images in memory\ntrain = pd.read_csv('../input/train.csv', nrows=10000)\nlabels = train.iloc[:,0].values.astype('int32')\nX_train = (train.iloc[:, 1:].values).astype('float32') # Pixel intensities of the images\nX_test = (pd.read_csv('../input/test.csv').values).astype('float32')\n\ny_train = np_utils.to_categorical(labels) # Labels as categories\n\n# Convert images to black and white\n#X_train[X_train > 0] = 1\nscale = np.max(X_train)\nX_train /= scale\nX_test /= scale\n\nmean = np.std(X_train)\nX_train -= mean\nX_test -= mean\n\n# Split data for training and validation.\nX_train, X_val, y_train, y_val = train_test_split(X_train, \n                                                  y_train, \n                                                  test_size=0.2, \n                                                  random_state=1)\n\nprint('X_train shape:', X_train.shape, '| y_train shape:', y_train.shape)\nprint('X_val shape:', X_val.shape, '| y_val shape:', y_val.shape)", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "fd2776417abd4d1aec722698de36f0fb8e36752e", "_cell_guid": "b5f4090c-75cf-4767-ac51-b28172495c6d"}, "execution_count": 6}, {"source": "## Multi-layer Perceptron using scikit-learn <a name=\"sklearn-mlp\"></a>", "outputs": [], "cell_type": "markdown", "metadata": {"_uuid": "c77e036fb84f99536f54c7f3c2822ae7640ef7e9", "_cell_guid": "dac8a03b-9c57-4054-9681-a2dae1dbfb7b"}, "execution_count": null}, {"source": "# Instantiate the MLP classifier\nclf = MLPClassifier(solver='lbfgs', # Limited-memory BFGS optimizer\n                    activation='relu', # Rectified linear unit as activation function, returns f(x) = max(0, x)\n                    hidden_layer_sizes=(500, ) # One hidden layer of 500 nodes \n                   )\n\n# Fit the model on the training set\nclf.fit(X_train, y_train)", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "ef288ed26d704d86282c7f482e7f8b3d0ee491fe", "_cell_guid": "f33665d5-3a4b-4e9b-b24f-9b95c22dbc8b"}, "execution_count": 7}, {"source": "# Run prediction on the validation set \ny_pred = clf.predict(X_val)\n\n# Print results\nprint('Classification report:\\n\\n', classification_report(y_val, y_pred), '\\n')", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "1612f8e7c5382de3d7caeb1d27a6ba1ac84bf5b8", "_cell_guid": "c59e32d1-797d-49ef-8681-28a64c15841b"}, "execution_count": 9}, {"source": "# Insert our prediction in a DataFrame with the ImageId\ny_pred = y_pred.argmax(1)\npredictions = pd.DataFrame({'ImageId': list(range(1, y_pred.size+1)), \n                            'Label': y_pred.astype(int)}\n                     )\nprint('Completed prediction of {} images'.format(predictions.shape[0]))", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "e73f9217d8214e8d974095206e87665cdbeefff3", "_cell_guid": "53060ddb-86bc-4839-a86e-84d964c1443f"}, "execution_count": 25}, {"source": "# Create figure\nfig = plt.figure(figsize=(9,6))\n\n# loop over the first 10 images of the test set\nfor i in range(25):\n    ax = plt.subplot(5, 5, i+1)\n    ax.set_title('Prediction: {}'.format(predictions.iloc[i].Label))\n    ax.axis('off')\n    image = X_val[i].reshape(28, 28)\n    imgplot = plt.imshow(image, cmap='binary')\n\nplt.show()", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "11f892233706c81888aa4cfc7e3c92ad1b6a79bc", "_cell_guid": "466a9cb1-5e05-4daa-b38a-55e4a7d8aeb5"}, "execution_count": 66}, {"source": "## Multi-layer Perceptron using Keras <a name=\"keras\"></a>", "outputs": [], "cell_type": "markdown", "metadata": {"collapsed": true, "_uuid": "62c701df1bc546a31c5f04f8d4f8480b00f31aa8", "_cell_guid": "cd708436-4e7c-44b3-b954-1ed0b6ac2fb6"}, "execution_count": null}, {"source": "input_dim = X_train.shape[1]\nnb_classes = y_train.shape[1]\n\n# Here's a Deep Dumb MLP (DDMLP)\nmodel = Sequential()\nmodel.add(Dense(128, input_dim=input_dim))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(128))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(nb_classes))\nmodel.add(Activation('softmax'))\n\n# we'll use categorical xent for the loss, and RMSprop as the optimizer\nmodel.compile(loss='categorical_crossentropy', optimizer='rmsprop')\n\nprint(\"Training...\")\nmodel.fit(X_train, y_train, \n          nb_epoch=10, \n          batch_size=16, \n          validation_split=0.2, \n#          show_accuracy=True, \n          verbose=2)\nprint('------------------------------\\n')\nprint(\"Training complete.\")", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "7abeeb7dd1f31e98c2a14296664301255997f4a0", "_cell_guid": "c36bd30e-c9e4-4cfb-a5e9-154a08a4379c"}, "execution_count": 68}, {"source": "print(\"Generating test predictions...\")\ny_pred = model.predict_classes(X_test, verbose=0)\n#print(\"Complete.\")\n\npredictions = pd.DataFrame({'ImageId': list(range(1, y_pred.size+1)), \n                            'Label': y_pred.astype(int)}\n                     )\nprint('Completed prediction of {} images'.format(predictions.shape[0]))", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "99c6d55dbfb80541839f643b7a12ffe55e91be66", "_cell_guid": "4e588174-00b4-4b98-8e83-d279ce591a94"}, "execution_count": 75}, {"source": "# Create figure\nfig = plt.figure(figsize=(9,6))\n\n# Loop over some images of the test set\nfor i in range(40):\n    ax = plt.subplot(8, 5, i+1)\n    ax.set_title('Prediction: {}'.format(predictions.iloc[i].Label))\n    ax.axis('off')\n    image = X_test[i].reshape(28, 28)\n    imgplot = plt.imshow(image, cmap='binary')\n\nplt.show()", "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "trusted": false, "_uuid": "0931904b106ae1db3d33c1848032d9df9c48d83f", "_cell_guid": "3fcd9882-9bee-4d86-889f-50d22ad43517"}, "execution_count": 78}], "metadata": {"language_info": {"pygments_lexer": "ipython3", "nbconvert_exporter": "python", "version": "3.6.1", "file_extension": ".py", "codemirror_mode": {"version": 3, "name": "ipython"}, "name": "python", "mimetype": "text/x-python"}, "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}}