{"cells":[{"metadata":{"_uuid":"f2156d1dd26a1243e18512002e10872c5bd7271e","_cell_guid":"d4c08f48-fe23-4ddb-ac46-d97f05397514"},"cell_type":"markdown","source":"# Digit Recognizer: Small DenseNet"},{"metadata":{"_execution_state":"idle","_uuid":"72334cb006d02a4bcfc2a2fe622524eba824c6f8","_cell_guid":"f67b9393-8ea1-4e23-b856-2ce149cfe421","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Model, Input, load_model\nfrom keras.layers import Activation, Concatenate, AveragePooling2D, GlobalAveragePooling2D\nfrom keras.layers import Dense, Conv2D, Flatten, MaxPool2D\nfrom keras.layers import Dropout, BatchNormalization\nfrom keras.layers.convolutional import UpSampling2D\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nfrom keras.initializers import RandomNormal\nfrom keras.applications import DenseNet121\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Flatten\nfrom keras.optimizers import RMSprop\n\nsns.set(style='white', context='notebook', palette='deep')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting execution conditions"},{"metadata":{"trusted":true},"cell_type":"code","source":"### Kaggle or Local-PC ###\nKAGGLE = True       # <==== SET ============\nif KAGGLE:\n    DIR = '../input'\nelse:               # local PC\n    DIR = './'","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"86061d98eccaa02efe0dab0fa3884e71fcf4c310","_cell_guid":"6d2fb3e6-ab71-4974-b5a2-4af1ebdb99f4"},"cell_type":"markdown","source":"## Data preparation"},{"metadata":{"_execution_state":"idle","_uuid":"84bbd5ab8d7895bd430d5ecfe2f7ddf77baa7b74","_cell_guid":"5e51d00e-62fd-4141-bf73-50ac4f2da7d0","trusted":true},"cell_type":"code","source":"# Load the data\ntrain = pd.read_csv(os.path.join(DIR, \"train.csv\"))\ntest = pd.read_csv(os.path.join(DIR, \"test.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"1213b979d5ed3e0d13824d17d694c79d2ece92fa","_cell_guid":"86570a36-5c20-460a-9dfd-2070548532a7","trusted":true},"cell_type":"code","source":"Y_train = train[\"label\"]\n\n# Drop 'label' column\nX_train = train.drop(labels = [\"label\"], axis = 1) \n\n# free some space\ndel train","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"b5d4f8fcf2a967e2c7d57daedf95aa8c5ab7f8cb","_cell_guid":"cdc4340b-6e24-4e12-be99-ac806098ff17","trusted":true},"cell_type":"code","source":"# Normalize the data\nX_train = X_train / 255.0\ntest = test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 28\nN_ch = 1","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"f0a6ad80dab8e0f2c2e46165ccd9cd82dd162bc3","_cell_guid":"34b6a5f7-8fd2-4387-8ef4-c9dc19584fed","trusted":true},"cell_type":"code","source":"# Reshape image in 3 dimensions (height, width, canal)\nX_train = X_train.values.reshape(-1,SIZE,SIZE,N_ch)\ntest = test.values.reshape(-1,SIZE,SIZE,N_ch)","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"cabefd1478d5c1bdfe57fd6a34395340916a854c","_cell_guid":"4b7f3e78-44dc-4561-b1f0-9429ee024cf4","trusted":true},"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\nY_train = to_categorical(Y_train, num_classes = 10)","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"b779ac76d8317647db92d5a88b4098d212d72884","_cell_guid":"dcd25ebb-d845-4d32-9867-082e352b1396","trusted":true},"cell_type":"code","source":"# Split the train and the validation set for the fitting\nX_train, X_val, Y_train, Y_val = train_test_split(\n    X_train, Y_train, test_size = 0.1, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fa18b37a9acd9e098bac1d12264b0dd4310fdd3","_cell_guid":"d5265777-aeb3-449d-b171-d88cad74c0a4"},"cell_type":"markdown","source":"# CNN(DenseNet)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dense block\ndef dense_block(input_tensor, input_channels, n_block, growth_rate):\n    x = input_tensor\n    n_ch = input_channels\n    for i in range(n_block):\n        # main root\n        main = x\n        # DenseBlock root\n        x = BatchNormalization()(x)\n        x = Activation(\"relu\")(x)\n        # Bottle-Neck 1x1 Convolution\n        x = Conv2D(128, (1, 1))(x)\n        x = BatchNormalization()(x)\n        x = Activation(\"relu\")(x)\n        # 3x3 Convolution\n        x = Conv2D(growth_rate, (3, 3), padding=\"same\")(x)\n        # Concatenate\n        x = Concatenate()([main, x])\n        n_ch += growth_rate\n    return x, n_ch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transition Layer\ndef transition_layer(input_tensor, input_channels, compression):\n    n_ch = int(input_channels * compression)\n    # 1x1 compression\n    x = Conv2D(n_ch, (1, 1))(input_tensor)\n    # AveragePooling\n    x = AveragePooling2D((2, 2))(x)\n    return x, n_ch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_denseNet(input_shape, growth_rate, blocks, compression):\n    input = Input(shape = input_shape)\n\n    n = growth_rate\n    x = Conv2D(n, (1,1))(input)\n\n    for i in range(len(blocks)):\n        # Transition layer\n        if i != 0:\n            x, n = transition_layer(x, n, compression)\n        # DenseBlock\n        x, n = dense_block(x, n, blocks[i], growth_rate)\n\n    x = GlobalAveragePooling2D()(x)\n#     x = Dense(512, activation=\"relu\")(x)\n    output = Dense(10, activation = \"softmax\")(x)\n\n    # model\n    model = Model(input,output)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_denseNet(input_shape = (SIZE, SIZE, N_ch),\n                       growth_rate = 16,    # 32\n                       blocks = [2,4,8,5],  # [2,4,8,5],[3,6,12,8],[6,12,24,16]\n                       compression = 0.5)\n# compile\nmodel.compile(optimizer = Adam(lr=0.001),\n              loss = \"categorical_crossentropy\",\n              metrics = [\"accuracy\"])\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"c4a5b4e462ec5362c47eef4fcc7956fd4e203307","_cell_guid":"b5987a18-6bbe-42a2-9d31-333ebc4f7af1","trusted":true},"cell_type":"code","source":"#Callback : \n\n# Set a learning rate annealer\nreduceLR = ReduceLROnPlateau(monitor = 'val_loss', \n                             patience = 2,\n                             factor = 0.5,\n                             min_lr = 1e-5,\n                             verbose = 1)\n# Save best model\nchkPoint = ModelCheckpoint('dense.h5',\n                           monitor = 'val_accuracy',\n                           save_best_only = True,\n                           save_weights_only = False,\n                           mode = 'auto',\n                           period = 1,\n                           verbose = 1)\n# Early stop\nearlyStop = EarlyStopping(monitor = 'val_accuracy',\n                          mode = 'auto',\n                          patience = 5,\n                          verbose = 1,\n                          min_delta = 0)","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"26b0647c46efdb6b1096cf7335a7bf2a3417543a","_cell_guid":"970db455-b393-4b25-806d-92c6766c12c0","trusted":true},"cell_type":"code","source":"epochs = 30\nbatch_size = 64     # 86","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"21d6192c87d92d497c797656474bccd9cefc5647","_cell_guid":"b342befe-1a6f-44bf-8dab-28033a729122","trusted":true},"cell_type":"code","source":"# With data augmentation to prevent overfitting (accuracy 0.99286)\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f24df64b223e0177c94025b6767ab19b722c5386","_cell_guid":"241a0f4f-13f5-4b13-be1e-4e3e4a714c06"},"cell_type":"markdown","source":"## Train"},{"metadata":{"_execution_state":"idle","_uuid":"cf36b3d029f95b553be02d612e097a9769ee8252","_cell_guid":"b453af8d-9736-43e3-b486-7a1cd7dd8909","trusted":true},"cell_type":"code","source":"# Fit the model\nhistory = model.fit_generator(datagen.flow(X_train,Y_train,\n                                               batch_size=batch_size),\n                                  epochs = epochs,\n                                  validation_data = (X_val,Y_val),\n                                  verbose = 1,\n                                  steps_per_epoch=X_train.shape[0] // batch_size,\n                                  callbacks=[reduceLR, chkPoint, earlyStop])\n# Save hist\ndf = pd.DataFrame(history.history)\ndf.to_csv('hist.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"3a831860dd5bb65c8ead1ddcf4ae18ae20dd7f3e","_cell_guid":"eb4b1b73-cbd4-40e8-9790-066fcef4c4c0","trusted":true},"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(2,1)\nax[0].plot(df['loss'], color='b', label=\"Training loss\")\nax[0].plot(df['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(df['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(df['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63698d7d51381b33892ce164b0f21930abb3e937","_cell_guid":"de1c65bd-4a88-4351-9f4b-562e72e7e0fd"},"cell_type":"markdown","source":"## predict and Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict results\nresults = model.predict(test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\n\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"_execution_state":"idle","_uuid":"369dfaab09240f3f12bcff91953ffd315ab84985","_cell_guid":"b5f1f39f-13b8-439a-8913-0f120e3d47a9","trusted":true},"cell_type":"code","source":"submission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","name":"python3","display_name":"Python 3"},"language_info":{"pygments_lexer":"ipython3","mimetype":"text/x-python","nbconvert_exporter":"python","version":"3.6.1","codemirror_mode":{"version":3,"name":"ipython"},"file_extension":".py","name":"python"}},"nbformat":4,"nbformat_minor":1}