{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.simplefilter(action='ignore', category=(RuntimeWarning, FutureWarning, UserWarning))\n\nimport time\nimport os\nimport numpy as np\nimport pandas as pd\nimport math\nimport gc\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom PIL import Image\nimport itertools\nfrom itertools import islice\nimport tensorflow as tf\nimport keras_tuner as kt\nimport keras\nfrom keras import layers, Model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout, MaxPooling2D, GlobalAvgPool2D, Input, concatenate, Average, RandomZoom, RandomTranslation, Rescaling, Concatenate, GlobalAveragePooling2D\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom keras.optimizers import SGD, Adam, Nadam, RMSprop\nfrom keras.utils import to_categorical, plot_model\nfrom keras.losses import categorical_crossentropy, SparseCategoricalCrossentropy\nfrom keras.metrics import categorical_accuracy\nfrom keras.applications import  InceptionResNetV2, MobileNet,  EfficientNetB3\nfrom IPython.display import Markdown, display\nfrom sklearn.preprocessing import LabelEncoder\n\n# to make this notebook's output stable across runs\nnp.random.seed(42)\ntf.random.set_seed(42)\n\nIMAGE_SIZE = (448, 448)\nBATCH_SIZE = 32","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-08-13T15:18:51.994970Z","iopub.execute_input":"2023-08-13T15:18:51.995417Z","iopub.status.idle":"2023-08-13T15:18:52.047362Z","shell.execute_reply.started":"2023-08-13T15:18:51.995378Z","shell.execute_reply":"2023-08-13T15:18:52.045974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folder = f\"/kaggle/input/dog-breed-identification/train/\"\ntest_folder = f\"/kaggle/input/dog-breed-identification/test/\"\nlabels_df = pd.read_csv(\"/kaggle/input/dog-breed-identification/labels.csv\")\nsubmission_df = pd.read_csv(\"/kaggle/input/dog-breed-identification/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:27.894279Z","iopub.execute_input":"2023-08-13T15:12:27.895587Z","iopub.status.idle":"2023-08-13T15:12:28.537482Z","shell.execute_reply.started":"2023-08-13T15:12:27.895544Z","shell.execute_reply":"2023-08-13T15:12:28.536536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:30.399219Z","iopub.execute_input":"2023-08-13T15:12:30.399771Z","iopub.status.idle":"2023-08-13T15:12:30.416175Z","shell.execute_reply.started":"2023-08-13T15:12:30.399726Z","shell.execute_reply":"2023-08-13T15:12:30.415222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:32.681889Z","iopub.execute_input":"2023-08-13T15:12:32.682243Z","iopub.status.idle":"2023-08-13T15:12:32.712978Z","shell.execute_reply.started":"2023-08-13T15:12:32.682213Z","shell.execute_reply":"2023-08-13T15:12:32.711976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = keras.Sequential([\n    layers.RandomFlip(\"horizontal\"), \n    layers.RandomRotation(0.05),\n    RandomZoom(0.1),\n], name='data_augmentation')","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:34.656307Z","iopub.execute_input":"2023-08-13T15:12:34.656656Z","iopub.status.idle":"2023-08-13T15:12:37.491407Z","shell.execute_reply.started":"2023-08-13T15:12:34.656628Z","shell.execute_reply":"2023-08-13T15:12:37.490430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_df['label'] = LabelEncoder().fit_transform(labels_df.breed)\nlabels_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:42.088919Z","iopub.execute_input":"2023-08-13T15:12:42.089270Z","iopub.status.idle":"2023-08-13T15:12:42.104928Z","shell.execute_reply.started":"2023-08-13T15:12:42.089241Z","shell.execute_reply":"2023-08-13T15:12:42.103965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_list = labels_df['label'].to_numpy().tolist()","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:44.697907Z","iopub.execute_input":"2023-08-13T15:12:44.698262Z","iopub.status.idle":"2023-08-13T15:12:44.703949Z","shell.execute_reply.started":"2023-08-13T15:12:44.698233Z","shell.execute_reply":"2023-08-13T15:12:44.702783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_breeds_df = labels_df[['label','breed']].copy()\nlabels_breeds_df.drop_duplicates(inplace=True)\nlabels_breeds_df.set_index('label',drop=True,inplace=True)\nindex_to_breed = labels_breeds_df.to_dict()['breed']\ndog_breeds = sorted(list(index_to_breed.values()))","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:38:32.642233Z","iopub.execute_input":"2023-08-13T15:38:32.642595Z","iopub.status.idle":"2023-08-13T15:38:32.656975Z","shell.execute_reply.started":"2023-08-13T15:38:32.642560Z","shell.execute_reply":"2023-08-13T15:38:32.656052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_raw_ds(train_folder, labels_list, split_size=0.1, image_size=(224,224)):\n    train_ds_raw, val_ds_raw = tf.keras.utils.image_dataset_from_directory(\n        train_folder,\n#         seed=SEED,\n        labels=labels_list,\n        label_mode='categorical',\n        validation_split=0.1,\n        shuffle=False,\n        subset=\"both\",\n        image_size=image_size,\n        batch_size=None,\n        crop_to_aspect_ratio=False\n    )\n    return train_ds_raw, val_ds_raw","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:50.824793Z","iopub.execute_input":"2023-08-13T15:12:50.825143Z","iopub.status.idle":"2023-08-13T15:12:50.831388Z","shell.execute_reply.started":"2023-08-13T15:12:50.825109Z","shell.execute_reply":"2023-08-13T15:12:50.830347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create final datasets for training after using the Keras tuner.","metadata":{}},{"cell_type":"code","source":"train_ds_raw, val_ds_raw = create_raw_ds(train_folder, labels_list, image_size=IMAGE_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:12:54.554483Z","iopub.execute_input":"2023-08-13T15:12:54.554847Z","iopub.status.idle":"2023-08-13T15:13:15.040478Z","shell.execute_reply.started":"2023-08-13T15:12:54.554810Z","shell.execute_reply":"2023-08-13T15:13:15.039525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds_raw = tf.keras.utils.image_dataset_from_directory(\n    test_folder,\n#     seed=SEED,\n    labels=None,\n    shuffle=False,\n    image_size=IMAGE_SIZE,\n    batch_size=None,\n    crop_to_aspect_ratio=False\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:14:51.071489Z","iopub.execute_input":"2023-08-13T15:14:51.071888Z","iopub.status.idle":"2023-08-13T15:15:24.202924Z","shell.execute_reply.started":"2023-08-13T15:14:51.071858Z","shell.execute_reply":"2023-08-13T15:15:24.201977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = train_ds_raw.batch(BATCH_SIZE).prefetch(buffer_size=tf.data.AUTOTUNE)\nval_ds = val_ds_raw.batch(BATCH_SIZE)\ntest_ds = test_ds_raw.batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:15:36.350142Z","iopub.execute_input":"2023-08-13T15:15:36.350537Z","iopub.status.idle":"2023-08-13T15:15:36.364952Z","shell.execute_reply.started":"2023-08-13T15:15:36.350504Z","shell.execute_reply":"2023-08-13T15:15:36.363897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = EarlyStopping(\n    monitor='val_categorical_accuracy', \n    patience=10,\n    verbose=1,\n    restore_best_weights = True)\nreduce_lr_on_plateau = ReduceLROnPlateau(\n    monitor='val_categorical_accuracy', \n    patience=3, verbose=1, factor=0.3\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:17:33.083696Z","iopub.execute_input":"2023-08-13T15:17:33.084076Z","iopub.status.idle":"2023-08-13T15:17:33.090163Z","shell.execute_reply.started":"2023-08-13T15:17:33.084047Z","shell.execute_reply":"2023-08-13T15:17:33.089203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_search = {\n    \n    'InceptionResNetV2': [InceptionResNetV2, keras.applications.inception_resnet_v2.preprocess_input],\n    'MobileNetV2': [MobileNet, keras.applications.mobilenet.preprocess_input],\n    'EfficientNetB3': [EfficientNetB3, keras.applications.efficientnet.preprocess_input]\n\n }","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:22:25.652706Z","iopub.execute_input":"2023-08-13T15:22:25.653090Z","iopub.status.idle":"2023-08-13T15:22:25.658715Z","shell.execute_reply.started":"2023-08-13T15:22:25.653062Z","shell.execute_reply":"2023-08-13T15:22:25.657475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_for_stacking = ['MobileNetV2', 'EfficientNetB3', 'InceptionResNetV2']","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:22:27.484118Z","iopub.execute_input":"2023-08-13T15:22:27.484483Z","iopub.status.idle":"2023-08-13T15:22:27.489012Z","shell.execute_reply.started":"2023-08-13T15:22:27.484455Z","shell.execute_reply":"2023-08-13T15:22:27.487957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_keras_model_feat(app_class, shape, inputs, prep_input_fx):\n    base_model = app_class(\n        include_top=False,\n        weights='imagenet',\n        input_shape=shape,\n    )\n    x = prep_input_fx(inputs)\n    x = base_model(x)\n    outputs = keras.layers.GlobalAveragePooling2D()(x)\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:22:31.051692Z","iopub.execute_input":"2023-08-13T15:22:31.052593Z","iopub.status.idle":"2023-08-13T15:22:31.061799Z","shell.execute_reply.started":"2023-08-13T15:22:31.052553Z","shell.execute_reply":"2023-08-13T15:22:31.060881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_feat_model(models_names, shape, aug_layer=None):\n    all_outputs = []\n    inputs = keras.Input(shape=shape)\n    if aug_layer != None:\n        aug_inputs = aug_layer(inputs)\n    else:\n        aug_inputs = inputs\n    for model_type in models_names:\n        model_class = models_search[model_type][0]\n        model_prep_input = models_search[model_type][1]\n        model_outputs = get_keras_model_feat(model_class, shape, aug_inputs, model_prep_input)\n        all_outputs.append(model_outputs)\n    concat_outputs = Concatenate()(all_outputs)\n    model = Model(inputs, concat_outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:22:32.939034Z","iopub.execute_input":"2023-08-13T15:22:32.939443Z","iopub.status.idle":"2023-08-13T15:22:32.946306Z","shell.execute_reply.started":"2023-08-13T15:22:32.939411Z","shell.execute_reply":"2023-08-13T15:22:32.945374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_features = build_feat_model(models_for_stacking, IMAGE_SIZE + (3,), data_augmentation)\nmodel_features.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:22:35.577837Z","iopub.execute_input":"2023-08-13T15:22:35.578505Z","iopub.status.idle":"2023-08-13T15:23:04.338759Z","shell.execute_reply.started":"2023-08-13T15:22:35.578473Z","shell.execute_reply":"2023-08-13T15:23:04.331787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_from_model(model, ds, count=1):\n    features = []\n    ds_has_y = isinstance(ds.element_spec, tuple)\n    if ds_has_y:\n        labels = []\n        y = np.concatenate([y for x, y in ds], axis=0)\n    for i in range(count):\n        predictions = model.predict(ds)\n        features.append(predictions)\n        if ds_has_y:\n            labels.append(y)\n    if ds_has_y:\n        return np.concatenate(features), np.concatenate(labels)\n    else:\n        return np.concatenate(features)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:23:22.730684Z","iopub.execute_input":"2023-08-13T15:23:22.731039Z","iopub.status.idle":"2023-08-13T15:23:22.738123Z","shell.execute_reply.started":"2023-08-13T15:23:22.731010Z","shell.execute_reply":"2023-08-13T15:23:22.737225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_train, y_train = get_features_from_model(model_features, train_ds, count=2)\nfeatures_val, y_val = get_features_from_model(model_features, val_ds)\nprint(features_train.shape, y_train.shape, features_val.shape, y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:23:24.747946Z","iopub.execute_input":"2023-08-13T15:23:24.748502Z","iopub.status.idle":"2023-08-13T15:30:58.330885Z","shell.execute_reply.started":"2023-08-13T15:23:24.748462Z","shell.execute_reply":"2023-08-13T15:30:58.329914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MCDropout(tf.keras.layers.Dropout):\n    def call(self, inputs, training=False):\n        return super().call(inputs, training=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:33:37.798099Z","iopub.execute_input":"2023-08-13T15:33:37.798737Z","iopub.status.idle":"2023-08-13T15:33:37.805655Z","shell.execute_reply.started":"2023-08-13T15:33:37.798694Z","shell.execute_reply":"2023-08-13T15:33:37.804337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dnn_model = keras.models.Sequential([\n    Input(features_train.shape[1:]),\n    MCDropout(0.7),\n    Dense(120, activation='softmax')\n])\ndnn_model.compile(\n    optimizer=keras.optimizers.Adam(0.001),\n    loss=categorical_crossentropy,\n    metrics=[\"categorical_accuracy\"]\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:33:39.113701Z","iopub.execute_input":"2023-08-13T15:33:39.114072Z","iopub.status.idle":"2023-08-13T15:33:39.199610Z","shell.execute_reply.started":"2023-08-13T15:33:39.114042Z","shell.execute_reply":"2023-08-13T15:33:39.198715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_dnn = dnn_model.fit(\n    features_train, \n    y_train,\n    validation_data=(features_val, y_val),\n    batch_size=128,\n    epochs=50,\n    callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-08-13T15:33:41.163283Z","iopub.execute_input":"2023-08-13T15:33:41.163797Z","iopub.status.idle":"2023-08-13T15:33:55.368897Z","shell.execute_reply.started":"2023-08-13T15:33:41.163756Z","shell.execute_reply":"2023-08-13T15:33:55.367875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_test = get_features_from_model(model_features, test_ds)\nfeatures_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:34:40.228745Z","iopub.execute_input":"2023-08-13T15:34:40.229114Z","iopub.status.idle":"2023-08-13T15:38:32.640326Z","shell.execute_reply.started":"2023-08-13T15:34:40.229084Z","shell.execute_reply":"2023-08-13T15:38:32.639247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_probas = np.stack([dnn_model.predict(features_test)\n                     for sample in range(80)])\npredictions = y_probas.mean(axis=0)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-08-13T15:38:49.444659Z","iopub.execute_input":"2023-08-13T15:38:49.445042Z","iopub.status.idle":"2023-08-13T15:40:36.571900Z","shell.execute_reply.started":"2023-08-13T15:38:49.445015Z","shell.execute_reply":"2023-08-13T15:40:36.570856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.loc[:, dog_breeds] = predictions\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-13T15:41:18.550062Z","iopub.execute_input":"2023-08-13T15:41:18.550456Z","iopub.status.idle":"2023-08-13T15:41:19.052227Z","shell.execute_reply.started":"2023-08-13T15:41:18.550424Z","shell.execute_reply":"2023-08-13T15:41:19.051144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}