{"cells":[{"metadata":{"_uuid":"5c942644dd3eeee24598318badc6f1f37c3b3486"},"cell_type":"markdown","source":"* [](http://)VGG16  com Regularization + Data Augmentation + Dropout 0.5"},{"metadata":{"_cell_guid":"8a7bff8d-c95a-4cfb-8c38-14ab989f769b","_uuid":"4da5a3e7db32799ca0108576e469157094d23111","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nnp.random.seed(2019)\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport keras\nfrom keras import regularizers\nfrom keras.applications.vgg16 import VGG16\nfrom keras.models import Model\nfrom keras.layers import Dense, Dropout, Flatten\n\n\nimport os\nfrom tqdm import tqdm\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nimport cv2\nfrom keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0b8fddbc-447f-4083-88c4-9bd5f811253d","_uuid":"7c64ad63b95b36df5a2223b4c499c1caad9c73ee"},"cell_type":"markdown","source":"Reading csv's to get the names of the images and their breeds."},{"metadata":{"_cell_guid":"bcbeef91-f00e-4a68-b1b8-25e206027bb2","_uuid":"18e9091e9a3d851fb0ce796a3b537ffb7f47c874","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/dog-breed-identification/labels.csv') # arquivo com os nomes das imagens e as suas raças para o treino\ndf_test = pd.read_csv('../input/dog-breed-identification/sample_submission.csv') # arquivo com os nomes das imagens de teste","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6abe20dc-a411-4334-ad66-2526d99c7f63","_uuid":"0ba75a32f91650bc75440443ab3d11c35e44e1c9","trusted":true,"scrolled":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ee13aba5a660346ec51f77240c410f6cb0170b6"},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"31789e79-005d-4219-82f7-b757f6595adc","_uuid":"860b860b8f98c9fe381645044628cc7492eaa6fd"},"cell_type":"markdown","source":"Races must be using * one-hot encode *"},{"metadata":{"_cell_guid":"1ba5da72-9d82-404d-aa25-ce9739a45775","_uuid":"7fbb53109a3d08bedb1abac074f8186bdea991ac","trusted":true},"cell_type":"code","source":"targets_series = pd.Series(df_train['breed'])\none_hot = pd.get_dummies(targets_series, sparse = True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"23fdc238-9f15-41d0-be8b-1913e6b1dd0f","_uuid":"07c3bf3155c29a76d6f42af9b2135bbbaca87c92","trusted":true},"cell_type":"code","source":"one_hot_labels = np.asarray(one_hot)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"421885e6-97a5-4a43-bdf5-6b60cfc0b3d3","_uuid":"d59fb275c2e30a410c9c8af8489f38138b1513ac"},"cell_type":"markdown","source":"The following will read the training and test images."},{"metadata":{"_cell_guid":"0e89e3ca-7c9e-4f76-aaf8-8c5267e3a415","_uuid":"7b47e6524b0fcf5ccb9a237cbdc8d444ff215dfb","trusted":true},"cell_type":"code","source":"im_size = 197","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cc632b19-d7c8-4c78-b5bf-4f0313603c11","_uuid":"e7f00cc68bbb5b9b15ee3373361bab01287476ce","trusted":true},"cell_type":"code","source":"x_train = []\ny_train = []\nx_test = []","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"98aab483-199c-451b-80a9-a9c1cf3d02be","_uuid":"1f099a6a4d6b16ed3202a91d5376b77579664d04","trusted":true},"cell_type":"code","source":"i = 0 \nfor f, breed in tqdm(df_train.values):\n    img = cv2.imread('../input/dog-breed-identification/train/{}.jpg'.format(f))\n    x_train.append(cv2.resize(img, (im_size, im_size)))\n    label = one_hot_labels[i]\n    y_train.append(label)\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"179ac0707e2bc96673e17b27a85a0a1226fa9155"},"cell_type":"code","source":"# delete df_train to decrease memory usage\ndel df_train","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ceaf5e6d-058d-495f-80d0-55b02870a936","_uuid":"bdee8455dd9f5f6232e57a83d19f06527b259611","trusted":true,"scrolled":true},"cell_type":"code","source":"for f in tqdm(df_test['id'].values):\n    img = cv2.imread('../input/dog-breed-identification/test/{}.jpg'.format(f))\n    x_test.append(cv2.resize(img, (im_size, im_size)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"60707a5f-0689-4ecf-a0df-3e3e9c76e076","_uuid":"00fdfb9bf52e9f7a8030d88dfab823262c0f1dc2","trusted":true},"cell_type":"code","source":"num_class = 120","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fd2dec22-e057-433b-81ba-f1d8a7580b8e","_uuid":"116f3d718f6bf256eb0a2992c344e5f52f9be181","trusted":true},"cell_type":"code","source":"X_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train,shuffle=True,  test_size=0.2, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b21a18f3d0a3610eb45f47fd44bc45d3702aa188"},"cell_type":"code","source":"datagen = ImageDataGenerator(width_shift_range=0.2,\n                            height_shift_range=0.2,\n                            zoom_range=0.2,\n                            rotation_range=30,\n                            vertical_flip=False,\n                            horizontal_flip=True)\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f49d7c1b5580b5b45c0ab95cb00272e96fb84d1"},"cell_type":"code","source":"train_generator = datagen.flow(np.array(X_train), np.array(Y_train), \n                               batch_size=32) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6900c96a1fe65084e0304e397f5c0f30b5de748e"},"cell_type":"code","source":"def create_my_model(use_regularizer, optimizer):\n    base_model = VGG16(weights=\"../input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\",include_top=False, input_shape=(im_size, im_size, 3))\n    dropout = base_model.output\n    dropout = Dropout(0.5)(dropout)\n    model_with_dropout = Model(inputs=base_model.input, outputs=dropout)\n        \n    x = base_model.output\n    x = Flatten()(x)\n    predictions = Dense(num_class, activation='softmax', kernel_regularizer=regularizers.l2(0.0015), activity_regularizer=regularizers.l1(0.0015))(x)\n    \n    my_model = Model(inputs=base_model.input, outputs=predictions)\n    \n#     for layer in my_model.layers:\n#         layer.treinable = False\n    \n    my_model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n    return my_model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec56f221a1d3c3fa811e2030eb1bfe69d6e57a04"},"cell_type":"code","source":"def gerar_grafico(historia, titulo):\n    plt.plot(historia.history['acc'])\n    plt.plot(historia.history['val_acc'])\n    plt.title('Accuracy ' + titulo)\n    plt.ylabel('Accuracy')\n    plt.xlabel('Times')\n    plt.legend(['training', 'validation'], loc='upper left')\n    plt.show()\n    plt.plot(historia.history['loss'])\n    plt.plot(historia.history['val_loss'])\n    plt.title('Loss ' + titulo)\n    plt.ylabel('Loss')\n    plt.xlabel('Times')\n    plt.legend(['training', 'validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee2252725ed633c358f6a3ad463844f399a4feaf"},"cell_type":"code","source":"model_rmsprop_com_regularizador = create_my_model(use_regularizer=True, optimizer='rmsprop')\nmodel_sgd_com_regularizador = create_my_model(use_regularizer=True, optimizer='sgd')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e05980052df17b4fa6fcb0f7656e153d38884d1a"},"cell_type":"code","source":"history_rmsprop_com_regularizador = model_rmsprop_com_regularizador.fit_generator(\n    train_generator,\n    epochs=10, steps_per_epoch=len(X_train) / 18, #len(X_train) / 18,\n    validation_data=(np.array(X_train), np.array(Y_train)), validation_steps=len(X_valid) / 18)\n\npreds = model_rmsprop_com_regularizador.predict(np.array(x_test), verbose=1)\n\ngerar_grafico(history_rmsprop_com_regularizador, \n              \"VGG16 with RMSprop\")\n\nsub = pd.DataFrame(preds)\ncol_names = one_hot.columns.values\nsub.columns = col_names\nsub.insert(0, 'id', df_test['id'])\nsub.head(5)\n\nsub.to_csv(\"predictions_vgg16_with_RMSProp.csv\")\n\nmodel_rmsprop_com_regularizador.save('vgg16_with_RMSProp.h5')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5333698bf27d3307b9b4535e345a2e58c8ad8b7d"},"cell_type":"code","source":"history_sgd_com_regularizador = model_sgd_com_regularizador.fit_generator(\n    train_generator,\n    epochs=10, steps_per_epoch=len(X_train) / 18, #len(X_train) / 18,\n    validation_data=(np.array(X_train), np.array(Y_train)), validation_steps=len(X_valid) / 18)\n\npreds = model_sgd_com_regularizador.predict(np.array(x_test), verbose=1)\n\ngerar_grafico(history_sgd_com_regularizador, \n              \"VGG16 com SGD\")\n\nsub = pd.DataFrame(preds)\ncol_names = one_hot.columns.values\nsub.columns = col_names\nsub.insert(0, 'id', df_test['id'])\nsub.head(5)\n\nsub.to_csv(\"output_sgd_v2_com_data_augmentation_e_sem_regularizador.csv\")\n\nmodel_sgd_com_regularizador.save('sgd_v2_com_data_augmentation_e_sem_regularizador.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\nf1_score(Y_train, Y_valid, average='macro') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score(Y_train, Y_valid, average='micro') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score(Y_train, Y_valid, average='weighted') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score(Y_train, Y_valid, average=None) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfrom sklearn import datasets\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import LinearSVC\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import (brier_score_loss, precision_score, recall_score,\n                             f1_score)\nfrom sklearn.calibration import CalibratedClassifierCV, calibration_curve\nfrom sklearn.model_selection import train_test_split\n\n\n# Create dataset of classification task with many redundant and few\n# informative features\n#X, y = datasets.make_classification(n_samples=100000, n_features=20,\n                                    #n_informative=2, n_redundant=10,\n                                    #random_state=42)\n#X_train, X_valid, Y_train, Y_valid # => \n#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.99,\n                                                    #random_state=42)\n\n\ndef plot_calibration_curve(est, name, fig_index):\n    \"\"\"Plot calibration curve for est w/o and with calibration. \"\"\"\n    # Calibrated with isotonic calibration\n    isotonic = CalibratedClassifierCV(est, cv=2, method='isotonic')\n\n    # Calibrated with sigmoid calibration\n    sigmoid = CalibratedClassifierCV(est, cv=2, method='sigmoid')\n\n    # Logistic regression with no calibration as baseline\n    lr = LogisticRegression(C=1.)\n\n    fig = plt.figure(fig_index, figsize=(10, 10))\n    ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)\n    ax2 = plt.subplot2grid((3, 1), (2, 0))\n\n    ax1.plot([0, 1], [0, 1], \"k:\", label=\"Perfectly calibrated\")\n    for clf, name in [(lr, 'Logistic'),\n                      (est, name),\n                      (isotonic, name + ' + Isotonic'),\n                      (sigmoid, name + ' + Sigmoid')]:\n        clf.fit(X_train, Y_train)\n        y_pred = clf.predict(X_valid)\n        if hasattr(clf, \"predict_proba\"):\n            prob_pos = clf.predict_proba(X_valid)[:, 1]\n        else:  # use decision function\n            prob_pos = clf.decision_function(X_valid)\n            prob_pos = \\\n                (prob_pos - prob_pos.min()) / (prob_pos.max() - prob_pos.min())\n\n        clf_score = brier_score_loss(Y_valid, prob_pos, pos_label=y.max())\n        print(\"%s:\" % name)\n        print(\"\\tBrier: %1.3f\" % (clf_score))\n        print(\"\\tPrecision: %1.3f\" % precision_score(Y_valid, y_pred))\n        print(\"\\tRecall: %1.3f\" % recall_score(Y_valid, y_pred))\n        print(\"\\tF1: %1.3f\\n\" % f1_score(Y_valid, y_pred))\n\n        fraction_of_positives, mean_predicted_value = \\\n            calibration_curve(Y_valid, prob_pos, n_bins=10)\n\n        ax1.plot(mean_predicted_value, fraction_of_positives, \"s-\",\n                 label=\"%s (%1.3f)\" % (name, clf_score))\n\n        ax2.hist(prob_pos, range=(0, 1), bins=10, label=name,\n                 histtype=\"step\", lw=2)\n\n    ax1.set_ylabel(\"Fraction of positives\")\n    ax1.set_ylim([-0.05, 1.05])\n    ax1.legend(loc=\"lower right\")\n    ax1.set_title('Calibration plots  (reliability curve)')\n\n    ax2.set_xlabel(\"Mean predicted value\")\n    ax2.set_ylabel(\"Count\")\n    ax2.legend(loc=\"upper center\", ncol=2)\n\n    plt.tight_layout()\n\n# Plot calibration curve for Gaussian Naive Bayes\nplot_calibration_curve(GaussianNB(), \"Naive Bayes\", 1)\n\n# Plot calibration curve for Linear SVC\nplot_calibration_curve(LinearSVC(max_iter=10000), \"SVC\", 2)\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}