{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nfrom keras import layers ,regularizers\nfrom keras.layers import Input, Add, Dense, Activation,BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D,Dropout\nfrom keras.models import Model\nfrom sklearn.preprocessing import LabelBinarizer\n\nfrom keras.preprocessing.image import ImageDataGenerator # for Data augmentation\nfrom keras.callbacks import ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def loadData():\n    ''' read training and testing images and reshape it to (m,28*28) and also read training labels\n    \n    - parameters \n    \n        - no inputs\n\n        - outputs :\n            - trainingX : np array of shape (42000 ,28,28) contain training images \n            - trainingY : np array of shape (42000 ,) contain training data labels \n            - testX : np array of shape(28000,28,28)\n    '''\n    \n    trainPath = \"../input/train.csv\"\n    testPath = \"../input/test.csv\"\n    training_frame = pd.DataFrame(pd.read_csv(trainPath))\n    test_frame = pd.DataFrame(pd.read_csv(testPath))\n    trainingY = np.array(training_frame['label'])\n    index = [\"pixel\"+str(x) for x in range(784)] # generate list of columns indices of data frame to read pixels value by it \n    trainingX = np.array(training_frame[index])\n    trainingX= trainingX.reshape(trainingX.shape[0],28,28,1)\n    testX = np.array(test_frame[index])\n    testX = testX.reshape(testX.shape[0],28,28,1)\n    return trainingX , trainingY , testX\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d35a060d13581901912116326f4b2bc8a6d5f3b9","collapsed":true},"cell_type":"code","source":"trainingX , trainingY , testX = loadData()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aade795c84ae40f7d2bda6522cc43c5be2e7380e","collapsed":true},"cell_type":"code","source":"# checking the dimensions\nprint(\"training x : \",trainingX.shape)\nprint(\"training y : \",trainingY.shape)\nprint(\"test x : \",testX.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"28e046e46b09a4d280c91c9301537aef360584e4"},"cell_type":"code","source":"def oneHot(y):\n    ''' convert labels into one hot coding \n        parameters:\n            - input : list of labels \n            - output : np array of one hot encoding \n    '''\n    encoder = LabelBinarizer()\n    y = encoder.fit_transform(y)\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"feb3bc9d09abec8989efcc6fa2e199fe42ab0ccc"},"cell_type":"code","source":"# this fun plot example of our images data \ndef plot_figures(x,y,nrows = 3, ncols=4):\n    \"\"\"Plot random figures of data.\n\n    Parameters\n    ----------\n    x : Images\n    y : labels\n    ncols : number of columns of subplots wanted in the display\n    nrows : number of rows of subplots wanted in the figure\n    \"\"\"\n    # generate random indexs \n    indexs = random.sample(range(0, len(y)), nrows*ncols)\n    \n    x = x.reshape(x.shape[0],28,28)\n    fig, axeslist = plt.subplots(ncols=ncols, nrows=nrows,figsize=(10,10))\n    for ind,i in enumerate(indexs):\n        axeslist.ravel()[ind].imshow(x[i], cmap=plt.gray())\n        axeslist.ravel()[ind].set_title(y[i])\n        axeslist.ravel()[ind].set_axis_off()\n    #plt.tight_layout() # optional","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"563a967cae0c97056b43acd79e61fdfeae65b1ee","collapsed":true},"cell_type":"code","source":"plot_figures(x = trainingX , y = trainingY)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"40096ffe3fca2e7940898c0e76e2c8adf0a27392"},"cell_type":"code","source":"## build our model \ndef vgg16(trainingX , trainingY):\n    \"\"\" implmentation of VGG16 model\n    \n    args:\n        input:\n            - trainingX : np array contain training data \n            - trainingY : np array contain training labels with out one hot coding -we will do this later in this fuc - \n            \n        output:\n            - mode : our keras mode of VGG16 \n            - trainingY : return training y after one hot encoding\n    \n    Architecture :\n            -> conv3-64 -> conv3-64\n            -> maxpool \n            -> conv3-128 ->  conv3-128\n            -> maxpool\n            -> conv3-256 -> conv3-256 -> conv1-256\n            -> maxpool\n            -> conv3-512 -> conv3-512 -> conv1-512\n            -> maxpool\n            -> conv3-512 -> conv3-512 -> conv1-512\n            -> maxpool\n            -> FC100 'relu'\n            -> FC50 'relu'\n            -> FC10  'softmax'\n            \n    \"\"\"\n\n    # create input tensor \n    inputShape = trainingX.shape\n    modelInput = Input(inputShape[1:])\n    x = modelInput\n    print(x.shape)\n    # one hot encoding for our labels\n    trainingY = oneHot(trainingY)\n    \n    # normalization \n    trainingX = trainingX /255.0\n    print(trainingY.shape)\n    # build the model\n    \n    x = Conv2D(64,(3,3),padding=\"same\", input_shape=(28,28,1),activation = 'relu',strides=(1,1),name='conv1',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(64,(3,3),padding=\"same\",activation = 'relu',strides=(1,1),name='conv2',kernel_initializer='glorot_uniform')(x)\n    \n    x =   MaxPooling2D((3, 3), strides=(2, 2) ,padding =\"same\",name=\"maxpool1\")(x)\n    \n    x = Conv2D(128,(3,3),padding=\"same\",activation = 'relu',strides=(1,1),name='conv3',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(128,(3,3),padding=\"same\",strides=(1,1),name='conv4',kernel_initializer='glorot_uniform',kernel_regularizer=regularizers.l2(0.01))(x)\n    \n    x = MaxPooling2D((3, 3), strides=(2, 2) ,padding =\"same\",name=\"maxpool2\")(x)\n    \n    x = Conv2D(256,(3,3),padding=\"same\",strides=(1,1),activation = 'relu',name='conv5',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(256,(3,3),padding=\"same\",strides=(1,1),activation = 'relu',name='conv6',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(256,(1,1),padding=\"same\",strides=(1,1),activation = 'relu',name='conv7',kernel_initializer='glorot_uniform',kernel_regularizer=regularizers.l2(0.01))(x)\n    \n    x = MaxPooling2D((3, 3), strides=(2, 2) ,padding =\"valid\",name=\"maxpool3\")(x)\n\n    x = Conv2D(512,(3,3),padding=\"same\",strides=(1,1),activation = 'relu',name='conv8',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(512,(3,3),padding=\"same\",strides=(1,1),activation = 'relu',name='conv9',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(512,(1,1),padding=\"same\",strides=(1,1),activation = 'relu',name='conv10',kernel_initializer='glorot_uniform',kernel_regularizer=regularizers.l2(0.01))(x)\n    \n    x = MaxPooling2D((3, 3), strides=(2, 2) ,padding =\"same\",name=\"maxpool4\")(x)\n    \n    x = Conv2D(512,(3,3),padding=\"same\",strides=(1,1),activation = 'relu',name='conv11',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(512,(3,3),padding=\"same\",strides=(1,1),activation = 'relu',name='conv12',kernel_initializer='glorot_uniform')(x)\n    x = Conv2D(512,(1,1),padding=\"same\",strides=(1,1),activation = 'relu',name='conv13',kernel_initializer='glorot_uniform',kernel_regularizer=regularizers.l2(0.01))(x)\n    \n    x = MaxPooling2D((3, 3), strides=(2, 2) ,padding =\"same\",name=\"maxpool5\")(x)\n    \n    #flatten x to be FC in next layer\n    x = Flatten()(x)\n    \n    #fc layers\n    x = Dense(100,name=\"fc1\", kernel_initializer = 'glorot_uniform',activation='relu')(x)\n    x = Dense(100,name=\"fc2\", kernel_initializer = 'glorot_uniform',activation='relu')(x)\n    x = Dense(10,name=\"fc3\", kernel_initializer = 'glorot_uniform',activation='softmax')(x)\n\n    #model\n    model = Model(inputs = modelInput, outputs = x, name='vgg16')\n\n    return model,trainingY\n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d5af295ec9981caad1bfcc378eb94fcb9046079","collapsed":true},"cell_type":"code","source":"model,trainingY = vgg16(trainingX,trainingY)\nmodel.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"aa02188b32f9354d4eff7a35d319c8b4f38a611a"},"cell_type":"code","source":"# reduce LR \nlr_reduce = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae70d1ef98021df45ed03c692ddba2f41d16f03f","collapsed":true},"cell_type":"code","source":"# Data augmentation\naug  = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False,\n        data_format = 'channels_last')  # randomly flip images\naug.fit(trainingX)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4df75c39e571e4b253f1b168b1812f0c1edf434c"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrainingX, X_val, trainingY, Y_val = train_test_split(trainingX, trainingY, test_size = 0.1, random_state=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20e201e74ff80565e094dad5dba913c4d75b9fd4"},"cell_type":"code","source":"modelHistory =model.fit_generator(aug.flow(trainingX,\n                    trainingY,batch_size = 64),\n                    epochs = 40,\n                    validation_data=(X_val,Y_val), \n                    shuffle=True,\n                    verbose = 2,\n                    callbacks = [lr_reduce])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"993885c6b36b25aca539bdd07406f95ea1a9a899"},"cell_type":"code","source":"def plotModelHistory(modeHistory):\n    # summarize history for accuracy\n    history = modeHistory\n    plt.plot(history.history['acc'])\n    plt.plot(history.history['val_acc'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()\n    # summarize history for loss\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b84442a64048149a81be11b524bd5dd93e172ab8"},"cell_type":"code","source":"plotModelHistory(modelHistory)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fa370f7165b0988e0f772c819f67073a498bc8ce"},"cell_type":"code","source":"#save model\nmodel.save_weights('vgg16_wieghts.h5')\nmodel.save('vgg16-digitRecognizer_keras.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d8af094c8338e194ee8f6846571670909258f358"},"cell_type":"code","source":"# create array for testY\nm = testX.shape[0]\ntestY = np.zeros((m,1))\ntestY = model.predict(testX)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2d17a6d9a6d4bdf15e46ba4dd85415cd4defd55"},"cell_type":"code","source":"# plot some test image with there predicted label\nfinalY = np.argmax(testY,axis=1)\nplot_figures(testX,finalY,nrows = 3, ncols=4)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d500511a6b007b274ce56490d99566a7080031f5"},"cell_type":"code","source":"# save submission file \nframe = pd.DataFrame({'Label': finalY.T.squeeze()})\nframe = frame.reset_index(drop=True)\nframe.index += 1 \nframe.to_csv(\"Digite Recognize.csv\", index_label='ImageId')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c25e0f89ab06a8b23236a39845d52d0e9d32d782","collapsed":true},"cell_type":"code","source":"print(frame)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cb31e251756953cd287171e18e69421ca7653fc0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}