{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Keras -- MLPs on MNIST"},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.utils import np_utils \nfrom keras.datasets import mnist \nimport seaborn as sns\nfrom keras.initializers import RandomNormal","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"%matplotlib notebook\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport time\n\ndef plt_dynamic(x, vy, ty, ax, colors=['b']):\n    ax.plot(x, vy, 'b', label=\"Validation Loss\")\n    ax.plot(x, ty, 'r', label=\"Train Loss\")\n    plt.legend()\n    plt.grid()\n    fig.canvas.draw()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"# the data, shuffled and split between train and test sets \n(X_train, y_train), (X_test, y_test) = mnist.load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Number of training examples :\", X_train.shape[0], \"and each image is of shape (%d, %d)\"%(X_train.shape[1], X_train.shape[2]))\nprint(\"Number of training examples :\", X_test.shape[0], \"and each image is of shape (%d, %d)\"%(X_test.shape[1], X_test.shape[2]))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"# the input shape is 3 dimensional vector\n# for each image we have a (28*28) vector\n# convert the (28*28) vector into single dimensional vector of 1 * 784 \n\nX_train = X_train.reshape(X_train.shape[0], X_train.shape[1]*X_train.shape[2]) \nX_test = X_test.reshape(X_test.shape[0], X_test.shape[1]*X_test.shape[2]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# after converting the input images from 3d to 2d vectors\n\nprint(\"Number of training examples :\", X_train.shape[0], \"and each image is of shape (%d)\"%(X_train.shape[1]))\nprint(\"Number of training examples :\", X_test.shape[0], \"and each image is of shape (%d)\"%(X_test.shape[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# An example data point\nprint(X_train[0])","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"# if we observe the above matrix each cell is having a value between 0-255\n# before we move to apply machine learning algorithms lets try to normalize the data\n# X => (X - Xmin)/(Xmax-Xmin) = X/255\n\nX_train = X_train/255\nX_test = X_test/255","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"# example data point after normlizing\nprint(X_train[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# here we are having a class number for each image\nprint(\"Class label of first image :\", y_train[0])\n\n# lets convert this into a 10 dimensional vector\n# this conversion needed for MLPs \n\nY_train = np_utils.to_categorical(y_train, 10) \nY_test = np_utils.to_categorical(y_test, 10)\n\nprint(\"After converting the output into a vector : \",Y_train[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>  Softmax classifier  </h2>"},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"from keras.models import Sequential \nfrom keras.layers import Dense, Activation \n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"# some model parameters\n\noutput_dim = 10\ninput_dim = X_train.shape[1]\n\nbatch_size = 128 \nnb_epoch = 20","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"# start building a model\nmodel = Sequential()\nmodel.add(Dense(output_dim, input_dim=input_dim, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\nhistory = model.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test)) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" <h3>  MLP + Sigmoid activation + SGDOptimizer </h3>"},{"metadata":{"trusted":false},"cell_type":"code","source":"# Multilayer perceptron\n\nmodel_sigmoid = Sequential()\nmodel_sigmoid.add(Dense(512, activation='sigmoid', input_shape=(input_dim,)))\nmodel_sigmoid.add(Dense(128, activation='sigmoid'))\nmodel_sigmoid.add(Dense(output_dim, activation='softmax'))\n\nmodel_sigmoid.summary()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"model_sigmoid.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_sigmoid.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model_sigmoid.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"w_after = model_sigmoid.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.title(\"Weight matrices after model trained\")\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2>MLP + Sigmoid activation + ADAM </h2>"},{"metadata":{"trusted":false},"cell_type":"code","source":"model_sigmoid = Sequential()\nmodel_sigmoid.add(Dense(512, activation='sigmoid', input_shape=(input_dim,)))\nmodel_sigmoid.add(Dense(128, activation='sigmoid'))\nmodel_sigmoid.add(Dense(output_dim, activation='softmax'))\n\nmodel_sigmoid.summary()\n\nmodel_sigmoid.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_sigmoid.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model_sigmoid.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"w_after = model_sigmoid.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.title(\"Weight matrices after model trained\")\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2> MLP + ReLU +SGD </h2>"},{"metadata":{"trusted":false},"cell_type":"code","source":"model_relu = Sequential()\nmodel_relu.add(Dense(512, activation='relu', input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.062, seed=None)))\nmodel_relu.add(Dense(128, activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.125, seed=None)) )\nmodel_relu.add(Dense(output_dim, activation='softmax'))\n\nmodel_relu.summary()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"model_relu.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_relu.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model_relu.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"w_after = model_relu.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.title(\"Weight matrices after model trained\")\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2> MLP + ReLU + ADAM </h2>"},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"model_relu = Sequential()\nmodel_relu.add(Dense(512, activation='relu', input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.062, seed=None)))\nmodel_relu.add(Dense(128, activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.125, seed=None)) )\nmodel_relu.add(Dense(output_dim, activation='softmax'))\n\nprint(model_relu.summary())\n\nmodel_relu.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_relu.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model_relu.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"w_after = model_relu.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.title(\"Weight matrices after model trained\")\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2> MLP + Batch-Norm on hidden Layers + AdamOptimizer </2>"},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.layers.normalization import BatchNormalization\n\nmodel_batch = Sequential()\n\nmodel_batch.add(Dense(512, activation='sigmoid', input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.039, seed=None)))\nmodel_batch.add(BatchNormalization())\n\nmodel_batch.add(Dense(128, activation='sigmoid', kernel_initializer=RandomNormal(mean=0.0, stddev=0.55, seed=None)) )\nmodel_batch.add(BatchNormalization())\n\nmodel_batch.add(Dense(output_dim, activation='softmax'))\n\n\nmodel_batch.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_batch.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_batch.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model_batch.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"w_after = model_batch.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.title(\"Weight matrices after model trained\")\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2> 5. MLP + Dropout + AdamOptimizer </h2>"},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.layers import Dropout\n\nmodel_drop = Sequential()\n\nmodel_drop.add(Dense(512, activation='sigmoid', input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.039, seed=None)))\nmodel_drop.add(BatchNormalization())\nmodel_drop.add(Dropout(0.5))\n\nmodel_drop.add(Dense(128, activation='sigmoid', kernel_initializer=RandomNormal(mean=0.0, stddev=0.55, seed=None)) )\nmodel_drop.add(BatchNormalization())\nmodel_drop.add(Dropout(0.5))\n\nmodel_drop.add(Dense(output_dim, activation='softmax'))\n\n\nmodel_drop.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_drop.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model_drop.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"score = model_drop.evaluate(X_test, Y_test, verbose=0) \nprint('Test score:', score[0]) \nprint('Test accuracy:', score[1])\n\nfig,ax = plt.subplots(1,1)\nax.set_xlabel('epoch') ; ax.set_ylabel('Categorical Crossentropy Loss')\n\n# list of epoch numbers\nx = list(range(1,nb_epoch+1))\n\nvy = history.history['val_loss']\nty = history.history['loss']\nplt_dynamic(x, vy, ty, ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"w_after = model_drop.get_weights()\n\nh1_w = w_after[0].flatten().reshape(-1,1)\nh2_w = w_after[2].flatten().reshape(-1,1)\nout_w = w_after[4].flatten().reshape(-1,1)\n\n\nfig = plt.figure()\nplt.title(\"Weight matrices after model trained\")\nplt.subplot(1, 3, 1)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h1_w,color='b')\nplt.xlabel('Hidden Layer 1')\n\nplt.subplot(1, 3, 2)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=h2_w, color='r')\nplt.xlabel('Hidden Layer 2 ')\n\nplt.subplot(1, 3, 3)\nplt.title(\"Trained model Weights\")\nax = sns.violinplot(y=out_w,color='y')\nplt.xlabel('Output Layer ')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2> Hyper-parameter tuning of Keras models using Sklearn </h2>"},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"from keras.optimizers import Adam,RMSprop,SGD\ndef best_hyperparameters(activ):\n\n    model = Sequential()\n    model.add(Dense(512, activation=activ, input_shape=(input_dim,), kernel_initializer=RandomNormal(mean=0.0, stddev=0.062, seed=None)))\n    model.add(Dense(128, activation=activ, kernel_initializer=RandomNormal(mean=0.0, stddev=0.125, seed=None)) )\n    model.add(Dense(output_dim, activation='softmax'))\n\n\n    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"activ = ['sigmoid','relu']\n\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom sklearn.model_selection import GridSearchCV\n\nmodel = KerasClassifier(build_fn=best_hyperparameters, epochs=nb_epoch, batch_size=batch_size, verbose=0)\nparam_grid = dict(activ=activ)\n\n# if you are using CPU\n# grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid)\ngrid_result = grid.fit(X_train, Y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.3"}},"nbformat":4,"nbformat_minor":1}