{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom keras.preprocessing import image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **1**"},{"metadata":{},"cell_type":"markdown","source":"## **1.a**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"labels = pd.read_csv('../input/dog-breed-identification/labels.csv')\nsample_submission = pd.read_csv('../input/dog-breed-identification/sample_submission.csv')\nprint('training set has ' + str(len(labels)) + ' entries')\nprint('test set has ' + str(len(sample_submission)) + ' entries')\nprint('total number of entries: ' + str(len(sample_submission) + len(labels)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **1.b**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '../input/dog-breed-identification/train/'\nimage_paths_train = {}\nfor name in labels['id']:\n    image_paths_train[name]=Image.open(train_dir + name + '.jpg')\n    \ntest_dir = '../input/dog-breed-identification/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k, img in image_paths_train.items():\n    print('image dimensions: ' + str(img.size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = image_paths_train['000bec180eb18c7604dcecc8fe0dba07']\nprint('image mode: ' + str(img.mode) + ' (3 channels)')\nclasses = labels.groupby('breed')['breed'].count()\nprint('number of classes: ' + str(classes.shape[0]))\nprint('image dimensions is not constant between all images, so we need to change the dimensions to be the same')\nprint('we can use augmentation, like flipping the image or shifting it (which rearanges the pixels but the breed features are preserved)')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **1.c**"},{"metadata":{"trusted":true},"cell_type":"code","source":"classes.plot(kind='bar', figsize=(22,7)).set_ylabel('count per breed')\nplt.title(\"entries per class distribution\")\nplt.show()\nprint('the data is somewhat balanced, but there are breeds with more examples than others')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **1.d**"},{"metadata":{},"cell_type":"markdown","source":"we found a result using FNN, with 4 layers and Adam optimizer. \n* results: \n* loss: 1.03\n* acc: 0.917\n\nwe also found a result using validation Xception + inception.\n* results: \n* logloss: 0.07\n* acc: 0.975"},{"metadata":{},"cell_type":"markdown","source":"## **1.e**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_multiple_imgs(X,y=None,nrow=2,ncol=2,figsize=(13,7),preds=None,skip=0):\n    fig,ax = plt.subplots(nrows=nrow,ncols=ncol,figsize=figsize)\n    fig.subplots_adjust(hspace=0.1, wspace=0.1)\n    for i in range(nrow*ncol):\n        ax[i//ncol,i%ncol].imshow(X[skip+i],cmap='binary')\n        ax[i//ncol,i%ncol].set_xticks([])\n        ax[i//ncol,i%ncol].set_yticks([])\n        if preds is not None:\n            ax[i//ncol,i%ncol].text(0.85, 0.1, str(preds[skip+i]), transform=ax[i//ncol,i%ncol].transAxes,\n                                   color='green' if y[skip+i]==preds[skip+i] else 'red',weight='bold')\n            ax[i//ncol,i%ncol].text(0.05, 0.1, str(y[skip+i]), color='blue',transform=ax[i//ncol,i%ncol].transAxes,weight='bold')\n        elif y is not None:\n            ax[i//ncol,i%ncol].text(0.05, 0.1, str(y[skip+i]), color='blue',transform=ax[i//ncol,i%ncol].transAxes,weight='bold')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_breed_imgs(breed, df, dic):\n    df_breed = df[df['breed'] == breed]\n    breed_dic = { key: dic[key] for key in df_breed['id'] }\n    plot_multiple_imgs(list(breed_dic.values()) )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Breeds that are easily separable**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('golden retriever')\nplot_breed_imgs('golden_retriever', labels, image_paths_train)\nprint('pekinese')\nplot_breed_imgs('pekinese', labels, image_paths_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Breeds that are harder to distinguish**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('norfolk terrier')\nplot_breed_imgs('norfolk_terrier', labels, image_paths_train)\nprint('norwich terrier')\nplot_breed_imgs('norwich_terrier', labels, image_paths_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **2**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Activation, Dropout, BatchNormalization, Conv2D, MaxPool2D\nfrom keras.utils import to_categorical\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nfor idx in labels.index:\n    img_id = labels['id'][idx]\n    imgs.append(cv2.resize(cv2.imread(train_dir + img_id + '.jpg', cv2.IMREAD_UNCHANGED), (224,224)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **2.a**\n we will be using the train-test split validation strategy with 20% test"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **2.b**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32,(3,3),activation='relu',input_shape=(224,224,3)))\nmodel.add(Conv2D(32,(3,3),activation='relu'))\nmodel.add(Dropout(0.1))\nmodel.add(MaxPool2D())\nmodel.add(Conv2D(16,(3,3),activation='relu'))\nmodel.add(Conv2D(16,(3,3),activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(MaxPool2D())\nmodel.add(Flatten())\nmodel.add(Dense(120,activation='softmax'))\nmodel.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelBinarizer\n%env TF_KERAS = 1\n\nencoder = LabelBinarizer()\ntransfomed_label = encoder.fit_transform(labels.breed)\nX = np.asarray(imgs)\nY = np.asarray(transfomed_label)\nX_train, X_test, y_train, y_test = train_test_split( X, Y, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(X_train,y_train,validation_split=0.2,shuffle=True,epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_results(history):\n    fig, ax = plt.subplots(1,2,figsize=(12,4))\n    ax[0].plot(history.history['accuracy'])\n    ax[0].plot(history.history['val_accuracy'])\n    ax[0].set_title('Model accuracy')\n    ax[0].set_ylabel('Accuracy')\n    ax[0].set_xlabel('Epoch')\n    ax[0].legend(['Train', 'Test'], loc='upper left')\n    \n    # Plot training & validation loss values\n    ax[1].plot(history.history['loss'])\n    ax[1].plot(history.history['val_loss'])\n    ax[1].set_title('Model loss')\n    ax[1].set_ylabel('Loss')\n    ax[1].set_xlabel('Epoch')\n    ax[1].legend(['Train', 'Test'], loc='upper left')\n    plt.show()\nshow_results(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,accuracy_score,log_loss\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_cat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = model.predict(X_test)\npred_cat = np.argmax(preds,axis=1)\ny_cat = np.argmax(y_test,axis=1)\nprint('model accuracy on test set is: {0:.2f}%'.format(accuracy_score(y_cat,pred_cat)*100))\nsns.heatmap(confusion_matrix(y_cat,pred_cat),cmap='Greens',annot=True, fmt='d')\nplt.xlabel('Prediction')\nplt.ylabel('True label')\nplt.title('mnist Convolutional model \\n classification results on test set')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_cat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_max = np.max(preds, axis=1)\nidx=0\nfor vals in preds:\n    pred_max = np.max(vals)\n    pred_idx = np.argmax(vals)\n    if(pred_max == 1):\n        \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **2.c**\nPossible reasons why the model accuracy isn't good:\n* model is overfitted because of the complexity of the model\n* resizing the different images might result in distortion of features in the image\n* the loss in the test grows while it goes down in the train, so we could use early stopping to stop at a better time "},{"metadata":{},"cell_type":"markdown","source":"## **2.d**\nPrioritizing the reasons:\n1. the loss in the test grows while it goes down in the train, so we could use early stopping to stop at a better time \n2. model is overfitted because of the complexity of the model\n3. resizing the different images might result in distortion of features in the image"},{"metadata":{},"cell_type":"markdown","source":"we will try to get better results by using ksplit verification strategy and simplifying the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. implementing early stopping"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels['file_name'] = labels['id'] + '.jpg'\nlabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n    validation_split=0.2)\n\n\n\n# compute quantities required for featurewise normalization\n# (std, mean, and principal components if ZCA whitening is applied)\ndg_train = datagen.flow_from_dataframe(\n        subset='training',\n        dataframe=labels,\n        directory=train_dir,\n        x_col=\"file_name\",\n        y_col=\"breed\",\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='categorical')\n\ndg_test = datagen.flow_from_dataframe(\n        subset='validation',\n        dataframe=labels,\n        directory=train_dir,\n        x_col=\"file_name\",\n        y_col=\"breed\",\n        target_size=(224, 224),\n        batch_size=32,\n        class_mode='categorical') \n\n# fits the model on batches with real-time data augmentation:\nhistory2 = model.fit_generator(dg_train, validation_data=dg_test, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_results(history2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(x=np.asarray(imgs_test), y=labels_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_callbacks(description = 'run1', es_patience = 10, rlop_patience = 7, tb_base_logdir = './logs/'):\n    cp = ModelCheckpoint('best_modelweights{}.hp'.format(description), save_best_only=True)\n    es = EarlyStopping(patience=es_patience, monitor = 'val_accuracy')\n    rlop = ReduceLROnPlateau(patience=rlop_patience)\n    cb = [cp,es,rlop]\n    return cb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32,(3,3),activation='relu',input_shape=(224,224,3)))\nmodel.add(Conv2D(32,(3,3),activation='relu'))\nmodel.add(Dropout(0.1))\nmodel.add(MaxPool2D())\nmodel.add(Conv2D(16,(3,3),activation='relu'))\nmodel.add(Conv2D(16,(3,3),activation='relu'))\nmodel.add(Dropout(0.1))\nmodel.add(MaxPool2D())\nmodel.add(Flatten())\nmodel.add(Dense(120,activation='softmax'))\nmodel.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n    \nhistory = model.fit(X_train,y_train,validation_split=0.2,shuffle=True,epochs=10,callbacks=set_callbacks('initial_CNN_model_dog_breed1', es_patience=2))\nshow_results(history) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. simplifying the model to avoid overfitting"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(16,(3,3),activation='relu',input_shape=(224,224,3)))\nmodel.add(Dropout(0.1))\nmodel.add(MaxPool2D())\nmodel.add(Conv2D(32,(3,3),activation='relu'))\nmodel.add(Dropout(0.1))\nmodel.add(MaxPool2D())\nmodel.add(Flatten())\nmodel.add(Dense(120,activation='softmax'))\nmodel.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n    \nhistory = model.fit(X_train,y_train,validation_split=0.2,shuffle=True,epochs=10)\nshow_results(history) ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}