{"cells":[{"metadata":{"colab":{},"colab_type":"code","id":"qM8J82HXHM62","trusted":false,"_uuid":"e3a701a0a9ee6ca8fbee6773a5de5545c29df63b"},"cell_type":"code","source":"%matplotlib inline\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import Counter\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\nfrom keras.models import model_from_json\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# My seed\n\nseed = 42","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"TQD_ssbnHM7A","_uuid":"c59ec35ed3ab89b073ce64011c4696da6907d82f"},"cell_type":"markdown","source":"### Loading the training and test dataset"},{"metadata":{"colab":{},"colab_type":"code","id":"RqDoVsC9HM7D","trusted":false,"_uuid":"958ebfe33fccd6a6b4ffc5920f880404775c5fac"},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_test  = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":104},"colab_type":"code","id":"94QDTFGlHM7K","outputId":"23ad4d26-3d70-41d3-ff04-6e24105c767a","trusted":false,"_uuid":"9b0d9f042d02c6beceeb5631171881935ccaba54"},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":104},"colab_type":"code","id":"IJeAI_2UHM7V","outputId":"83ebd3f0-8821-4994-e06f-d87aab1686a8","trusted":false,"_uuid":"21a99a59b3f8b92b7dfd844062aa3b5504e11d49"},"cell_type":"code","source":"df_test.info()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"xY3FuCv-HM7c","_uuid":"25aab3e7463c1d6951624b76265d41aef8ad6e34"},"cell_type":"markdown","source":"### Spliting the dataset"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":69},"colab_type":"code","id":"fTjseu2XHM7f","outputId":"5f92aa1d-2639-471f-ca02-2a509ecd4ad5","trusted":false,"_uuid":"92b8b5f06c17d765cbbfbbb7b0f7aea368654ea7"},"cell_type":"code","source":"X_train = df_train.drop(['label'], axis=1)\ny_train = df_train['label']\nX_test = df_test\n\n# Free memory space\n\ndel df_train\ndel df_test\n\nprint('Shape of X_train:', X_train.shape)\nprint('Shape of y_train:', y_train.shape)\nprint('Shape of X_test :', X_test.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"P67_rsamHM7l","_uuid":"d8902d2f9c7fca5ab8e0b27c50b639b6476e9872"},"cell_type":"markdown","source":"### Counting the labels"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":191},"colab_type":"code","id":"kRiEicH7HM7n","outputId":"04dc447b-5a7e-495f-8460-fba87a1b6676","trusted":false,"_uuid":"3bb13e70ba81307355b7142ef44733fdc692e080"},"cell_type":"code","source":"counter = Counter(y_train)\ncounter","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"1yB4I_nrHM7y","_uuid":"36d83a447f49bfdcaafb96362a83628053a74260"},"cell_type":"markdown","source":"### Normalizing the values of training and test"},{"metadata":{"colab":{},"colab_type":"code","id":"4WMu2MblHM71","trusted":false,"_uuid":"45abb215241fb69984b5e2779d5de443e461b026"},"cell_type":"code","source":"X_train = X_train / 255\nX_test = X_test / 255","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"zjZ13rBtHM7-","_uuid":"51b26aab4b061141f9596b135f070eb2de3b7a16"},"cell_type":"markdown","source":"### Reshape the images in 4 dimensions to use with Keras"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":52},"colab_type":"code","id":"fTllDgzdHM7_","outputId":"2ac9e51f-2c66-4dfe-fc9c-ef097261afcd","trusted":false,"_uuid":"973c22bf950b6f3d43a4c88140b87a119869c5c6"},"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1) # (height = 28px, width = 28px , canal = 1)\nX_test = X_test.values.reshape(-1,28,28,1)\n\nprint('Shape of X_train:', X_train.shape)\nprint('Shape of X_test :', X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"iiCE1sBnHM8J","_uuid":"6eaec55b9b3890f8106ff56088a45e9874ec8c81"},"cell_type":"markdown","source":"### Converting y values (labels) to categorical values"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","id":"FldleonhHM8J","outputId":"c1487d9b-2edb-4d43-ce7b-c979a6b56938","trusted":false,"_uuid":"e086342843e8cf1c6f08d24b5bee9d4373a1e556"},"cell_type":"code","source":"# One Hot Categories\n\ny_train = to_categorical(y_train, num_classes = 10)\ny_train.shape","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"0zX5q-yeHM8R","_uuid":"f9d15f081021fb355367dbb00d15f24588150394"},"cell_type":"markdown","source":"### Define the baseline neural network model"},{"metadata":{"colab":{},"colab_type":"code","id":"YZzy77A6HM8V","trusted":false,"_uuid":"4b565f974d99da814555277795271a5878ce97c8"},"cell_type":"code","source":"def baseline_model():\n    \n    # Create baseline\n    \n    baseline = Sequential()\n\n    #------------------------------------------------------------\n    \n    # Parameters tunned by GridSearchCV    \n    # 32 filters for the three firsts conv2D layers\n    \n    baseline.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu', \n                     input_shape = (28, 28, 1)))\n    baseline.add(BatchNormalization())\n    baseline.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu'))\n    baseline.add(BatchNormalization())\n    baseline.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu'))\n    baseline.add(BatchNormalization())\n    \n    # This layer simply acts as a downsampling filter. \n    # It looks at the 2 neighboring pixels and picks the maximal value, reducing computational cost, \n    # and to some extent also reduce overfitting.\n    \n    # IMPORTANT: Combining convolutional and pooling layers, CNN are able to combine local features and \n    # learn more global features of the image.\n    \n    baseline.add(MaxPool2D(pool_size=(2,2)))\n    \n    # Dropout is a regularization method, where a proportion of nodes (25%) in the layer are randomly ignored \n    # for each training sample. This dropout forces the network to learn features in a distributed way \n    # and improves generalization and reduces the overfitting.\n    \n    baseline.add(Dropout(0.25))\n    #------------------------------------------------------------\n    \n    # 64 filters for the three last conv2D layers\n    \n    baseline.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu'))\n    baseline.add(BatchNormalization())\n    baseline.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu'))\n    baseline.add(BatchNormalization())\n    baseline.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu'))\n    baseline.add(BatchNormalization())\n    \n    baseline.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n    baseline.add(Dropout(0.25))\n    #------------------------------------------------------------\n\n    # The Flatten layer is use to convert the final feature maps into a one single 1D vector. \n    # IMPORTANT: It combines all the found local features of the previous convolutional layers.\n    \n    baseline.add(Conv2D(filters = 128, kernel_size = (3,3), padding = 'Same', activation ='sigmoid'))\n    baseline.add(BatchNormalization())\n    \n    baseline.add(Flatten())\n    baseline.add(Dense(128, activation = \"relu\"))\n    baseline.add(Dropout(0.4))\n    \n    # The net outputs distribution of probability of each class --> In our case, 10 output classes\n    \n    baseline.add(Dense(10, activation = \"softmax\"))\n    \n    # The optimizer will iteratively improve parameters in order to minimize the loss.\n    # Compile the baseline including the optimizer and evaluating the performance of the baseline by accuracy\n    \n    baseline.compile(optimizer = 'Adamax' , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])\n    \n    return baseline","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"xVF-L9DtHM8b","_uuid":"a0bf07a874a55067d0903e4e9081030ac24fab53"},"cell_type":"markdown","source":"### Learning Rate"},{"metadata":{"colab":{},"colab_type":"code","id":"0RozIgRBHM8c","trusted":false,"_uuid":"498870e88213193f081caac62cc1c113760dfd5f"},"cell_type":"code","source":"# If after the third epoch we didn't have an improvement of accuracy, the learning rate will be \n# reduced by 50% (factor).\n\nlr_reduction = ReduceLROnPlateau(monitor='val_acc',\n                                 patience=3, \n                                 verbose=0, \n                                 factor=0.5, \n                                 min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"l_zBIykRHM8h","_uuid":"7007cb76415eb0ef2085c1b6b365992383d0abd8"},"cell_type":"markdown","source":"### Data augmentation"},{"metadata":{"colab":{},"colab_type":"code","id":"4gJaCXuIHM8u","trusted":false,"_uuid":"0379b3a74e5964401e779ed9c1c0e44d173ee2d5"},"cell_type":"code","source":"# The idea is to alter the training data with small transformations to reproduce the variations \n# occuring when someone is writing a digit. It's a way to minimize the overfitting of the model.\n\ngenerator = ImageDataGenerator(featurewise_center = False,\n                               samplewise_center = False, \n                               featurewise_std_normalization = False,\n                               samplewise_std_normalization = False,\n                               zca_whitening = False,\n                               rotation_range = 10, # Rotate image in 10 degrees\n                               zoom_range = 0.10, # Zoom image (10% zoom) \n                               width_shift_range = 0.10, # Shift image horizontally (10% of width)\n                               height_shift_range = 0.10, # Shift image vertically (10% of height)\n                               horizontal_flip = False,\n                               vertical_flip = False)\n\ngenerator.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"mCbwS1l4HM8x","_uuid":"88177036601c3d00b81a2feea6d36285a4e510d6"},"cell_type":"markdown","source":"### Creating 10 nets and training every ones"},{"metadata":{"colab":{},"colab_type":"code","id":"0qE_BCrQHM83","trusted":false,"_uuid":"16c9f7f9ec64ef31032b10ed659cb29ab5075985"},"cell_type":"code","source":"nets = 10\ndigits = [0] * nets\nhistory = [0] * nets\n\nepochs = 40\nbatch_size = 90","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":208},"colab_type":"code","id":"gv78_SCvHM9C","outputId":"68abe0c2-bde2-4a7f-8bb7-7610c9de0d90","trusted":false,"_uuid":"e460619d245e45c7ed0a0707958cf55cc75ca8ae"},"cell_type":"code","source":"print('Creating {0} CNNs...'.format(nets))\nfor model in range(nets):\n    digits[model] = baseline_model()\n    \n    # Splitting train and test datasets\n    \n    X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)\n    \n    history[model] = digits[model].fit_generator(generator.flow(X_train_aux,\n                                                              y_train_aux, \n                                                              batch_size = batch_size),\n                                                 epochs = epochs, \n                                                 steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                                 validation_data = (X_test_aux, y_test_aux), \n                                                 callbacks=[lr_reduction],\n                                                 verbose=0)\n    \n    print(\"CNN {0:>2d}: Epochs = {1:d}, Max. Train accuracy = {2:.5f}, Max. Validation accuracy = {3:.5f}\".format(\n        model + 1, # Number of the CNN model\n        epochs, # Total of epochs\n        max(history[model].history['acc']), # Maximum Accuracy from Training\n        max(history[model].history['val_acc']))) # Maximum Accuracy from Test (validation)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"-L5IBZyiHM9O","_uuid":"073b29a5a1d1e2b1a80ddfa5e67031c20dab4cfb"},"cell_type":"markdown","source":"### Getting the predictions with more probabilities to be correct"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":139},"colab_type":"code","id":"V2LVLlw0PHea","outputId":"2cdaa058-eed1-44a1-850f-bdff18feed7b","trusted":false,"_uuid":"c04a152f57273be1f92ab81b12c0267af81aa629"},"cell_type":"code","source":"label_predicted = np.zeros( (X_test.shape[0], 10) )\nlabel_predicted","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"PecSo_dfHM9S","trusted":false,"_uuid":"ffbcf64f43dc8856474767fc31e7f838ad1b1873"},"cell_type":"code","source":"for model in range(nets):\n    label_predicted = label_predicted + digits[model].predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"j21xpf3ZBWu3","trusted":false,"_uuid":"e81d73188695033f4c313cca17e0b8d50695aa6d"},"cell_type":"code","source":"# Get the index with the maximum probability\n\nlabel_predicted = np.argmax(label_predicted, axis = 1)\nlabel_predicted = pd.Series(label_predicted, name = \"Label\")","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"xparMvpNHM9e","trusted":false,"_uuid":"fdcaa031467f5ad3e68ab9ddde12cb690b2451da"},"cell_type":"code","source":"solution = pd.concat([pd.Series(range(1, 28001), name = \"ImageId\"), label_predicted], axis = 1)\nsolution.to_csv(\"solution_cnn_opt.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"colab_type":"code","id":"J9mS0wkYHM9k","outputId":"197fa169-b52e-4577-d245-672bdae81bfb","trusted":false,"_uuid":"50e8c2dfaca12444c472b1625b4b2ff673c87260"},"cell_type":"code","source":"solution.head(10)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"e5s6JHRoHM9s","_uuid":"5532114e503492fcb063eed588c020e108c199ed"},"cell_type":"markdown","source":"### Saving the models"},{"metadata":{"colab":{},"colab_type":"code","id":"ZwYT9t_lHM9t","trusted":false,"_uuid":"e51347184046bc6175b27d355584479feb6e06d4"},"cell_type":"code","source":"for model in range(nets):\n    model_saved = digits[model].to_json()\n    name = 'model_' + str(model) + '.json'\n    with open(name, 'w') as json_file:\n        json_file.write(model_saved)\n    name = 'model_' + str(model) + '.h5'\n    digits[model].save_weights(name)","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"9yrGrqZTHM9x","trusted":false,"_uuid":"0d44ea6a951e103be77dd08e23cb4ff841d7540d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":[],"name":"digits_v7.ipynb","provenance":[],"toc_visible":true,"version":"0.3.2"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}