{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Dropout\nfrom keras.utils import np_utils\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom sklearn.model_selection import GridSearchCV\nimport matplotlib.pyplot as plt\nfrom keras.datasets import mnist\nimport numpy as np\nnp.random.seed(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initialize variables "},{"metadata":{"trusted":true},"cell_type":"code","source":"number_of_epochs = 10\nb_size = 128\nnumber_of_classes = 10\nhidden_neurons = 256\nvalidation = 0.25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from numpy import loadtxt\n(x_train, y_train),(x_test, y_test) = mnist.load_data(path=\"mnist.npz\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ix = 0\nfor digit in x_train[:5]:\n    plt.title('{lable}'.format(lable=y_train[ix]))\n    plt.imshow(digit, cmap='gray')\n    plt.show()\n    ix += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape,x_test.shape, y_train.shape, y_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train[0].shape,x_test[0].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reshape data for Dense layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.reshape(x_train.shape[0],784)\nx_test  = x_test.reshape(x_test.shape[0],784)\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\nx_train = x_train/255\nx_test = x_test/255\n\ny_train = np_utils.to_categorical(y_train)\ny_test = np_utils.to_categorical(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train[0].shape,x_test[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (x_train[0])\nprint (y_train[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model(num_layers):\n    model = Sequential()\n    if num_layers==0:\n        model.add(Dense(output_dim = number_of_classes, input_dim = x_train.shape[1],activation = 'softmax'))\n        model.compile(loss='categorical_crossentropy', optimizer= 'sgd',metrics = ['accuracy'])\n        return model\n\n    for i in range(num_layers):\n        if i==0:\n            model.add(Dense(output_dim = hidden_neurons, input_dim = x_train.shape[1],activation = 'relu'))\n        else:\n            model.add(Dense(output_dim = hidden_neurons, activation = 'relu'))\n    model.add(Dense(output_dim = number_of_classes, activation = 'softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer= 'sgd',metrics = ['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(x_train.shape[1],)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model with 0 layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = make_model(num_layers=0)\nprint (model.summary())\nprint ('------------------------------------------------------------------------------')\nprint ('Training Model')\nhistory = model.fit(x_train, y_train, batch_size=b_size, epochs=number_of_epochs, validation_split=validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"]=(20, 10)\nfor key in history.history.keys():\n    plt.plot(range(1, number_of_epochs+1), history.history[key])\n\nplt.legend(list(history.history.keys()), loc=\"upper left\")\nplt.title(\"Learning Curve\")\nplt.xlabel('Epoch')\nplt.ylabel('Train / CV Error')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model with 5 layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = make_model(num_layers=5)\nprint (model.summary())\nprint ('------------------------------------------------------------------------------')\nprint ('Training Model')\nhistory = model.fit(x_train, y_train, batch_size=b_size, epochs=number_of_epochs, validation_split=validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"]=(20, 10)\nfor key in history.history.keys():\n    plt.plot(range(1, number_of_epochs+1), history.history[key])\n\nplt.legend(list(history.history.keys()), loc=\"upper left\")\nplt.title(\"Learning Curve\")\nplt.xlabel('Epoch')\nplt.ylabel('Train / CV Error')\nplt.show()  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model with 10 layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = make_model(num_layers=10 )\nprint (model.summary())\nprint ('------------------------------------------------------------------------------')\nprint ('Training Model')\nhistory = model.fit(x_train, y_train, batch_size=b_size, epochs=number_of_epochs*5, validation_split=validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"]=(20, 10)\nfor key in history.history.keys():\n    plt.plot(range(1, 5*number_of_epochs+1), history.history[key])\n\nplt.legend(list(history.history.keys()), loc=\"upper left\")\nplt.title(\"Learning Curve\")\nplt.xlabel('Epoch')\nplt.ylabel('Train / CV Error')\nplt.show()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sk_params = {\"num_layers\": [1,2]}\nestimator = KerasClassifier(build_fn = make_model, epochs = 1, batch_size = b_size, verbose = 2)\ngrid_search = GridSearchCV(estimator = estimator, param_grid = sk_params, n_jobs =1, verbose = 2, cv = 5)\ngrid_result = grid_search.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create the model with additional parameters: number of layers, number of hidden neurons and dropout"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_new_model(num_layers, num_hidden_neurons, dropout):\n    model = Sequential()\n    if num_layers==0:\n        model.add(Dense(output_dim = number_of_classes, input_dim = x_train.shape[1],activation = 'softmax'))\n        model.compile(loss='categorical_crossentropy', optimizer= 'sgd',metrics = ['accuracy'])\n        return model\n\n    for i in range(num_layers):\n        if i==0:\n            model.add(Dense(output_dim = num_hidden_neurons, input_dim = x_train.shape[1],activation = 'relu'))\n            model.add(Dropout(dropout))\n        else:\n            model.add(Dense(output_dim = num_hidden_neurons, activation = 'relu'))\n            model.add(Dropout(dropout))\n    model.add(Dense(output_dim = number_of_classes, activation = 'softmax'))\n    model.compile(loss='categorical_crossentropy', optimizer= 'sgd',metrics = ['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sk_params = {\"num_layers\": [1,2], \"num_hidden_neurons\": [128,256], \"dropout\":[0.3,0.5]}\nestimator = KerasClassifier(build_fn = make_new_model, epochs = 3, batch_size = b_size, verbose = 2)\ngrid_search = GridSearchCV(estimator = estimator, param_grid = sk_params, n_jobs =1, verbose = 2)\ngrid_result = grid_search.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_)) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# References\nhttps://www.youtube.com/watch?v=N-pugZsqjWY&fbclid=IwAR3HwYRmEy1M6VZ3Uz6zKIpxTuze94fMtaulOq29v2Q7FEfH-DfP_aq30a0"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":4}