{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install hyperas","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nnp.random.seed(0)\n\nfrom sklearn.model_selection import train_test_split\n\nfrom hyperopt import Trials, STATUS_OK, tpe\nfrom hyperas import optim\nfrom hyperas.distributions import choice, uniform\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.utils import to_categorical\nfrom keras import optimizers\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data():\n    train = pd.read_csv('../input/train.csv')\n    test = pd.read_csv('../input/test.csv')\n#     print(train.shape, test.shape)\n    # review\n    Y = train['label']\n    X = train.drop(axis=1,columns='label')\n    X = X.values.reshape(-1,28,28,1) # convert to ndarray\n    Y = Y.values\n    test = test.values.reshape(-1,28,28,1)\n#     print(X.shape)\n    # normalization\n    if np.max(X) > 1:\n        X = X / 255\n    if np.max(test) > 1:\n        test = test / 255\n#     print(np.max(X), np.max(test))\n    # categorization\n    if len(Y.shape) == 1:\n        num_classes = len(set(Y))\n        Y = to_categorical(Y, num_classes)\n#     print(num_classes)\n    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.33, random_state=0)\n    return X_train, X_test, y_train, y_test\n\n# data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = 10","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 1\n\nFirst model without hyper-parameters selection and too much wonder about architecture.<br>\ntraining loss, accuracy <br>\n0.0185, 0.9934 <br>\nvalidation loss, accuracy <br>\n0.0483, 0.9874 <br>\n\nThe train loss in the model is very low as compared to the validation set loss which tells us that the model has over-fit.\nSubmission accuracy: 0.98585"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1 = Sequential([\n    Conv2D(filters=32, kernel_size=(3, 3), input_shape=(28, 28, 1)),\n    MaxPool2D(pool_size=(2, 2)),\n    \n    Conv2D(filters=64, kernel_size=(3, 3), activation='relu'),\n    MaxPool2D(pool_size=(2, 2)),\n    \n    Conv2D(filters=128, kernel_size=(3, 3), activation='relu'),\n    Dropout(0.25),\n    \n    Conv2D(filters=256, kernel_size=(3, 3), activation='relu'),\n    Dropout(0.25),\n    \n    Flatten(),\n    Dense(512, activation='relu'),\n    \n    Dense(num_classes, activation='softmax')\n])\n\nmodel_1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel_1.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 2\n\nConv-Pool-Conv-Pool model - 2 layers with increasing number of filters and filter size. <br>\ntraining loss, accuracy <br>\n0.0137, 0.9959 <br>\nvalidation loss, accuracy <br>\n0.0507, 0.9848 <br>\n\nYet again model has over-fit.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2 = Sequential([\n    Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)),\n    MaxPool2D(pool_size=(2, 2), strides=2),\n    \n    Conv2D(filters=32, kernel_size=(5, 5), activation='relu'),\n    MaxPool2D(pool_size=(2, 2),  strides=2),\n    \n    Flatten(),\n    \n    Dense(128, activation='relu'),\n    Dense(64, activation='relu'),\n    Dense(num_classes, activation='softmax')\n])\n\nmodel_2.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel_2.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 3\n\nConv-Pool-Conv-Pool model - 3 layers with increasing number of filters. <br>\ntraining loss, accuracy <br>\n0.0103, 0.9967 <br>\nvalidation loss, accuracy <br>\n0.0414, 0.9877 <br>\n\nYet again model has over-fit but the difference in loss functions is smaller. After changing learning rate of optimizer the difference was even smaller but accuracy was lower on validation set."},{"metadata":{"trusted":true},"cell_type":"code","source":"model_3 = Sequential([\n    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)),\n    MaxPool2D(pool_size=(2, 2), strides=2),\n    \n    Conv2D(filters=64, kernel_size=(3, 3), activation='relu'),\n    MaxPool2D(pool_size=(2, 2),  strides=2),\n    \n    Conv2D(filters=128, kernel_size=(3, 3), activation='relu'),\n    MaxPool2D(pool_size=(2, 2),  strides=2),\n    \n    Flatten(),\n    \n    Dense(128, activation='relu'),\n    Dense(64, activation='relu'),\n    Dense(num_classes, activation='softmax')\n])\n\noptimizer = optimizers.Adam(lr=0.0001)\nmodel_3.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\nmodel_3.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 4\n\nConv-Conv-Pool-Conv-Conv-Pool - 2 layers with increasing number of filters, learning rate change to 0.0001, dropouts added. <br>\ntraining loss, accuracy <br>\n0.0569, 0.9822 <br>\nvalidation loss, accuracy <br>\n0.0515, 0.9836\n\nNo overfitting! Submission accuracy: 0.98957 <br>\nAccuracies of all models are quite the same - this model is a winner to further optimalization by hyper-parameter selection. "},{"metadata":{"trusted":true},"cell_type":"code","source":"model_4 = Sequential([\n    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)),\n    Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),\n    MaxPool2D(pool_size=(2, 2), strides=2),\n    Dropout(0.25),\n    \n    Conv2D(filters=64, kernel_size=(3, 3), activation='relu'),\n    Conv2D(filters=64, kernel_size=(3, 3), activation='relu'),\n    MaxPool2D(pool_size=(2, 2),  strides=2),\n    Dropout(0.25),\n    \n    Flatten(),\n    \n    Dense(256, activation='relu'),\n    Dense(128, activation='relu'),\n    Dense(num_classes, activation='softmax')\n])\n\noptimizer = optimizers.Adam(lr=0.0001)\nmodel_4.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\nmodel_4.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model_4\nmodel.fit(\n    X_train, y_train,\n    batch_size=128,\n    epochs=10,    \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model.evaluate(X_test, y_test, verbose=0)\nscore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(test)\npredictions = np.argmax(predictions, axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimize!\n\nI'm betting on model_4 but to feel the thrill I added also model_3 for optimization. Will the force be with Dropouts?"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model(X_train, Y_train, X_val, Y_val):\n    num_classes = 10\n    \n    model = Sequential()\n    model_choice = {{choice(['model_3', 'model_4'])}}\n    if model_choice == 'model_3':\n        model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)))\n        model.add(MaxPool2D(pool_size=(2, 2), strides=2))\n        \n        model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))\n        model.add(MaxPool2D(pool_size=(2, 2),  strides=2))\n        model.add(Dropout({{uniform(0, 1)}}))\n        \n        model.add(Conv2D(filters=128, kernel_size=(3, 3), activation='relu'))\n        model.add(MaxPool2D(pool_size=(2, 2),  strides=2))\n        model.add(Dropout({{uniform(0, 1)}}))\n        \n        model.add(Flatten())\n        \n        model.add(Dense(128), activation='relu')\n        model.add(Dense({{choice([32, 64])}}, activation='relu'))\n        \n    \n    elif model_choice == 'model_4':\n        model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)))\n        model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'))\n        model.add(MaxPool2D(pool_size=(2, 2), strides=2))\n        model.add(Dropout({{uniform(0, 1)}}))\n    \n        model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))\n        model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))\n        model.add(MaxPool2D(pool_size=(2, 2),  strides=2))\n        model.add(Dropout({{uniform(0, 1)}}))\n    \n        model.add(Flatten())\n        \n        model.add(Dense({{choice([128, 256])}}, activation='relu'))\n        model.add(Dense({{choice([64, 128])}}, activation='relu'))\n        \n    model.add(Dense(num_classes, activation='softmax'))\n    \n    adam = keras.optimizers.Adam(lr=0.0001)\n    \n    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam)\n    \n    model.fit(X_train, y_train,\n              batch_size=128,\n              nb_epoch=10,\n              verbose=2,\n              validation_data=(X_test, y_test))\n    \n    score, acc = model.evaluate(X_test, y_test, verbose=0)\n    print('Val accuracy:', acc)\n    return {'loss': -acc, 'status': STATUS_OK, 'model': model}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_run, best_model = optim.minimize(model=model,\n                                      data=data,\n                                      algo=tpe.suggest,\n                                      max_evals=30,\n                                      trials=Trials(),\n                                      notebook_name='CNN_first_steps')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'ImageId': range(1,len(test)+1) ,'Label':predictions })\n\nsubmission.to_csv(\"cnn_architecture.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resources\n\n1. Vladimir Alekseichenko, DataWorkshop course\n2. https://towardsdatascience.com/a-guide-to-an-efficient-way-to-build-neural-network-architectures-part-ii-hyper-parameter-42efca01e5d7"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}