{"nbformat_minor": 0, "cells": [{"execution_count": null, "source": "# Classification of MNIST Dataset using Deep Learning\n\nThis notebook is derived from Poonam's [solution][1] and Francois's [solution][2]. It contains two models, first is a fully connected neural network and the second is a convolutional neural network. \n\n[1]: https://www.kaggle.com/poonaml/deep-neural-network-keras-way\n[2]: https://www.kaggle.com/fchollet/simple-deep-mlp-with-keras/code/code", "cell_type": "markdown", "metadata": {"_uuid": "6521beb397a29632096b57b6c544b8d9109d5bcc", "_cell_guid": "84d4608d-4cc3-fcbb-57fb-61f07ad7d020", "_active": false}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# import libraries\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Sequential\nfrom keras.callbacks import EarlyStopping\nfrom keras.optimizers import Adam ,RMSprop\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.utils.np_utils import to_categorical\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Dense, Dropout, Lambda, Flatten, Activation, Dropout\n\n# fix random seed for reproducibility\nseed = 43\nnp.random.seed(seed)\n\n%matplotlib inline", "cell_type": "code", "metadata": {"_uuid": "f5ab98ef7ca9f1a6e28fe1d640ad9c8fa9cb4dbc", "trusted": false, "_cell_guid": "e5b02688-c589-5a89-e11c-837c6a99eb6e", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# Load Dataset", "cell_type": "markdown", "metadata": {"_uuid": "b5de97cfc97745ef27ac3d24c037859c3694f1d0", "_cell_guid": "22a7fd70-ab61-432d-24cb-93e558414495", "_active": false}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# loading training and test datasets\ntest= pd.read_csv(\"../input/test.csv\")\ntrain = pd.read_csv(\"../input/train.csv\")\n\n# create training and test labels\ny_train = train.iloc[:,0].values.astype('int32') \nX_train = (train.iloc[:,1:].values).astype('float32') \n\nX_test = test.values.astype('float32')", "cell_type": "code", "metadata": {"_uuid": "f3dfadbf115b97b037c8a5cdc8a380398140f21c", "trusted": false, "_cell_guid": "05226b08-226a-1a00-044d-a0e6b2101388", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# Visualize Dataset", "cell_type": "markdown", "metadata": {"_uuid": "772a300cda1471698876d6602d43e8af19e83ef9", "_cell_guid": "60957d82-c76f-4822-28ff-def7011a34fa", "_active": false}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "#Convert train datset to (num_images, img_rows, img_cols) format \nX_train = X_train.reshape(X_train.shape[0], 28, 28)\n\nfor i in range(6, 9):\n    plt.subplot(330 + (i+1))\n    plt.imshow(X_train[i], cmap=plt.get_cmap('gray'))\n    plt.title(y_train[i])    ", "cell_type": "code", "metadata": {"_uuid": "281d36ce89fd7abe605f994950f095757a584feb", "trusted": false, "_cell_guid": "1541678d-a08b-d2b2-1e1e-eabf882baaec", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# Preprocess Dataset\n\n- Convert to 2D arrays\n- Standardize dataset\n- One-hot encoding of image labels", "cell_type": "markdown", "metadata": {"_uuid": "2624f160781b9b8af13e7f2c990b18410d27e1ef", "_cell_guid": "1232c385-3cb2-56fd-4d1d-f027df7bc78e", "_active": false}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "#expand 1 more dimention as 1 for colour channel gray\nX_test = X_test.reshape(X_test.shape[0], 28, 28,1)\nX_train = X_train.reshape(X_train.shape[0], 28, 28, 1)\n\n# standardize dataset\nmean_px = X_train.mean().astype(np.float32)\nstd_px = X_train.std().astype(np.float32)\n\ndef standardize(x): \n    return (x-mean_px)/std_px\n\n# convert to one-hot encoding\ny_train = to_categorical(y_train)\nnum_classes = y_train.shape[1]", "cell_type": "code", "metadata": {"_uuid": "ae1d9d3fd9ab1448ccf86430c54de115c620fe71", "trusted": false, "_cell_guid": "a3f837ef-0373-8d91-46e6-30992cf73166", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"metadata": {"_uuid": "821802650474c6e01c47b25343aa9a33cc6dd701", "_cell_guid": "6a89dcdd-7b68-6ed1-2c39-b3a1edb3e7be", "_active": false}, "cell_type": "markdown", "execution_state": "idle", "source": "# Fully Connected Neural Network Model\n\n- Create model\n- Train model\n- Visualize performance\n- Perform test inference", "outputs": [], "execution_count": null}, {"execution_count": null, "source": "# no hidden layers\nmodel= Sequential()\nmodel.add(Lambda(standardize,input_shape=(28,28,1)))\nmodel.add(Flatten())\nmodel.add(Dense(350))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(100))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(num_classes, activation='softmax'))\nprint(\"input shape \",model.input_shape)\nprint(\"output shape \",model.output_shape)\n\nmodel.compile(optimizer=RMSprop(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])", "cell_type": "code", "metadata": {"_uuid": "7ce2a82009351ef560780276d788dd13f71b6ca0", "trusted": false, "_cell_guid": "a2c27783-3cfa-e907-4749-1e340a513f26", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# partition to train and val\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\n\n# create an image generator and batches\ngen = ImageDataGenerator()\nbatches = gen.flow(X_train, y_train, batch_size=16)\nval_batches=gen.flow(X_val, y_val, batch_size=16)\n\n# train the model and get performance data\nhistory=model.fit_generator(batches, batches.n, nb_epoch=1, validation_data=val_batches, \n                            nb_val_samples=val_batches.n)\nhistory_dict = history.history", "cell_type": "code", "metadata": {"_uuid": "baaab66dd93d1c9a48aaf6eccd1925dc78c20085", "trusted": false, "_cell_guid": "db3b4be6-4f72-c6cc-65cd-b45978db2462", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "epochs = range(1, len(loss_values)+1)\n\nacc_values = history_dict['acc']\nloss_values = history_dict['loss']\nval_acc_values = history_dict['val_acc']\nval_loss_values = history_dict['val_loss']\n\n# visualize accuracies\nplt.figure()\nplt.plot(epochs, loss_values, 'bo')\nplt.plot(epochs, val_loss_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.show()\n\n# visualize losses\nplt.figure()   \nplt.plot(epochs, acc_values, 'bo')\nplt.plot(epochs, val_acc_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.show()", "cell_type": "code", "metadata": {"_uuid": "f39bc28566342e153f4e279dcbf1fd58f4ffa73d", "trusted": false, "_cell_guid": "df40f5fc-586a-1fae-025e-ee508a8d9b71", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"execution_count": null, "source": "# apply predictions to model\npredictions = model.predict_classes(X_test, verbose=0)\n\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)), \"Label\": predictions})\nsubmissions.to_csv(\"fcnn.csv\", index=False, header=True)", "cell_type": "code", "metadata": {"_uuid": "986babfc210429a862e2b32057e5ec20eab3fcb8", "trusted": false, "_cell_guid": "c2841d54-f3dd-1ee8-a30d-4457dec0a67a", "_active": false, "_execution_state": "idle"}, "outputs": [], "execution_state": "idle"}, {"metadata": {"_uuid": "88c87df5c7537f8580a8cca7d7d6cdbbadf3fc7a", "collapsed": false, "_cell_guid": "87b85018-d6fd-4217-92a1-7bcc4b6808e0", "_execution_state": "idle"}, "outputs": [], "source": "# Convolutional Neural Network Model\n\n- Create model\n- Train model\n- Visualize performance\n- Perform test inference", "cell_type": "markdown", "execution_count": null}, {"metadata": {"_uuid": "2887bd5a78898cedcbd986264128db82a1cf1ba0", "collapsed": false, "_cell_guid": "cb9687cb-9638-4d9c-9c9f-f0282797c4fb", "_execution_state": "idle", "trusted": false}, "outputs": [], "source": "# no hidden layers\nmodel= Sequential()\nmodel.add(Lambda(standardize,input_shape=(28,28,1)))\nmodel.add(Conv2D(10,(3,3)))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(20,(3,3)))\nmodel.add(MaxPooling2D())\nmodel.add(Flatten())\nmodel.add(Dense(500))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(num_classes, activation='softmax'))\nprint(\"input shape \",model.input_shape)\nprint(\"output shape \",model.output_shape)\n\nmodel.compile(optimizer=RMSprop(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])", "cell_type": "code", "execution_count": null}, {"metadata": {"_uuid": "6aac6a332d0646ca0ef06039513cecaa3c90c55d", "collapsed": false, "_cell_guid": "085a60dc-1952-4ed0-82cc-06597e219065", "_execution_state": "idle", "trusted": false}, "outputs": [], "source": "# partition to train and val\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\n\n# create an image generator and batches\ngen = ImageDataGenerator()\nbatches = gen.flow(X_train, y_train, batch_size=16)\nval_batches=gen.flow(X_val, y_val, batch_size=16)\n\n# train the model and get performance data\nhistory=model.fit_generator(batches, batches.n, nb_epoch=1, validation_data=val_batches, \n                            nb_val_samples=val_batches.n)\nhistory_dict = history.history", "cell_type": "code", "execution_count": null}, {"metadata": {"_uuid": "7e4a017224a1ced50abc0a0126b175db74179e0a", "collapsed": false, "_cell_guid": "f99c7701-1b94-405e-8872-6225d5175324", "_execution_state": "idle", "trusted": false}, "outputs": [], "source": "epochs = range(1, len(loss_values)+1)\n\nacc_values = history_dict['acc']\nloss_values = history_dict['loss']\nval_acc_values = history_dict['val_acc']\nval_loss_values = history_dict['val_loss']\n\n# visualize accuracies\nplt.figure()\nplt.plot(epochs, loss_values, 'bo')\nplt.plot(epochs, val_loss_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.show()\n\n# visualize losses\nplt.figure()   \nplt.plot(epochs, acc_values, 'bo')\nplt.plot(epochs, val_acc_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.show()", "cell_type": "code", "execution_count": null}, {"metadata": {"_uuid": "cdba494da959d4d0b1e74090d3b36ecb23dec0c0", "collapsed": false, "_cell_guid": "f7cccf12-cf0e-45e9-a182-8e340a560f53", "_execution_state": "idle", "trusted": false}, "outputs": [], "source": "# apply predictions to model\npredictions = model.predict_classes(X_test, verbose=0)\n\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)), \"Label\": predictions})\nsubmissions.to_csv(\"cnn.csv\", index=False, header=True)", "cell_type": "code", "execution_count": null}], "metadata": {"language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "name": "python", "pygments_lexer": "ipython3", "version": "3.6.1"}, "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}, "nbformat": 4}