{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"IMG_SIZE = 128\nIMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)\nBATCH_SIZE = 256","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:01.824600Z","iopub.execute_input":"2023-06-11T01:02:01.825052Z","iopub.status.idle":"2023-06-11T01:02:01.829742Z","shell.execute_reply.started":"2023-06-11T01:02:01.825020Z","shell.execute_reply":"2023-06-11T01:02:01.828677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Carregar imagens","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv('/kaggle/input/dog-breed-identification/labels.csv')\nimage_train_path = '/kaggle/input/dog-breed-identification/train'","metadata":{"execution":{"iopub.status.busy":"2023-06-11T00:59:46.538764Z","iopub.execute_input":"2023-06-11T00:59:46.539418Z","iopub.status.idle":"2023-06-11T00:59:46.574820Z","shell.execute_reply.started":"2023-06-11T00:59:46.539377Z","shell.execute_reply":"2023-06-11T00:59:46.573946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-06-11T00:59:47.885613Z","iopub.execute_input":"2023-06-11T00:59:47.886299Z","iopub.status.idle":"2023-06-11T00:59:47.906503Z","shell.execute_reply.started":"2023-06-11T00:59:47.886266Z","shell.execute_reply":"2023-06-11T00:59:47.905476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Definir gerador para carregar os lotes preguiçosamente\nEvitar estouro de memória :(","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport math\nimport numpy as np\nfrom tensorflow.keras.utils import load_img\nfrom tensorflow.keras.utils import img_to_array\nfrom tensorflow.keras.applications.vgg19 import preprocess_input\n\n\nclass DogBreedGenerator(tf.keras.utils.Sequence):\n\n    def __init__(self, x_set, y_set, batch_size: int, resolution=IMG_SHAPE[:2]):\n        self.x, self.y = x_set, y_set\n        self.batch_size = batch_size\n        self.resolution = resolution\n\n    def __len__(self):\n        return math.ceil(len(self.x) / self.batch_size)\n\n    def __getitem__(self, index: int):\n        low = index * self.batch_size\n        high = min(low + self.batch_size, len(self.x))\n        batch_x = self.x[low:high]\n        batch_y = self.y[low:high]\n        final_batch_x = np.array([\n            preprocess_input(img_to_array(load_img(file_name, target_size=self.resolution)))\n            for file_name in batch_x\n        ])\n        return final_batch_x, np.array(batch_y)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:01:28.933643Z","iopub.execute_input":"2023-06-11T01:01:28.934082Z","iopub.status.idle":"2023-06-11T01:01:28.949878Z","shell.execute_reply.started":"2023-06-11T01:01:28.934047Z","shell.execute_reply":"2023-06-11T01:01:28.948958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import path\n\nall_image_paths = [path.join(image_train_path, id + '.jpg') for id in df.id]\nall_image_paths[:3]","metadata":{"execution":{"iopub.status.busy":"2023-06-11T00:59:51.468678Z","iopub.execute_input":"2023-06-11T00:59:51.469761Z","iopub.status.idle":"2023-06-11T00:59:51.500325Z","shell.execute_reply.started":"2023-06-11T00:59:51.469719Z","shell.execute_reply":"2023-06-11T00:59:51.499110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One-hot encoding","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom keras.utils import np_utils\n\nnum_classes = len(df.breed.unique())\nlabel_encoder = LabelEncoder()\nbreed = label_encoder.fit_transform(df.breed)\ntrain_y = np_utils.to_categorical(breed, num_classes=num_classes)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T00:59:53.015608Z","iopub.execute_input":"2023-06-11T00:59:53.016509Z","iopub.status.idle":"2023-06-11T00:59:53.321177Z","shell.execute_reply.started":"2023-06-11T00:59:53.016472Z","shell.execute_reply":"2023-06-11T00:59:53.320207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_generator = DogBreedGenerator(all_image_paths, train_y, BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:01:34.232234Z","iopub.execute_input":"2023-06-11T01:01:34.232648Z","iopub.status.idle":"2023-06-11T01:01:34.239024Z","shell.execute_reply.started":"2023-06-11T01:01:34.232617Z","shell.execute_reply":"2023-06-11T01:01:34.237849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelo","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications.vgg19 import VGG19\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Flatten, Dropout\n\nvgg_model = VGG19(weights='imagenet', include_top=False, input_shape=IMG_SHAPE)\n\nx = vgg_model.output\nx = GlobalAveragePooling2D()(x)\nx = Dropout(0.2)(x)\nout = Dense(num_classes, activation='softmax')(x)\n\nmodel = Model(inputs=vgg_model.input, outputs=out)\nvgg_model.trainable = False\n\nmetrics = ['accuracy']\nloss_name = 'categorical_crossentropy'\nmodel.compile(optimizer='adam', loss=loss_name, metrics=metrics)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:00:30.546490Z","iopub.execute_input":"2023-06-11T01:00:30.546936Z","iopub.status.idle":"2023-06-11T01:00:31.072083Z","shell.execute_reply.started":"2023-06-11T01:00:30.546898Z","shell.execute_reply":"2023-06-11T01:00:31.071252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Treinamento","metadata":{}},{"cell_type":"code","source":"history=model.fit(\n    data_generator, batch_size=BATCH_SIZE, epochs=20,\n    verbose=1, shuffle=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:02:05.907913Z","iopub.execute_input":"2023-06-11T01:02:05.908491Z","iopub.status.idle":"2023-06-11T01:12:44.596082Z","shell.execute_reply.started":"2023-06-11T01:02:05.908454Z","shell.execute_reply":"2023-06-11T01:12:44.595145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Teste","metadata":{}},{"cell_type":"code","source":"dataset_path = '/kaggle/input/dog-breed-identification'","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:12:44.598542Z","iopub.execute_input":"2023-06-11T01:12:44.598977Z","iopub.status.idle":"2023-06-11T01:12:44.603722Z","shell.execute_reply.started":"2023-06-11T01:12:44.598938Z","shell.execute_reply":"2023-06-11T01:12:44.602803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Copiar dataset de teste p/ um novo diretório\nJá que não é possível alterar o dataset original disponibilizado pelo Kaggle","metadata":{}},{"cell_type":"code","source":"!mkdir data\n!cp -r \"{dataset_path}/test\" data","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:12:44.605513Z","iopub.execute_input":"2023-06-11T01:12:44.606343Z","iopub.status.idle":"2023-06-11T01:14:08.514461Z","shell.execute_reply.started":"2023-06-11T01:12:44.606308Z","shell.execute_reply":"2023-06-11T01:14:08.513095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Renomear diretórios p/ ter a estrutura \"test/data\"\nEstrutura no padrão entendido pelo método flow_from_directory do tensorflow","metadata":{}},{"cell_type":"code","source":"!mv data test\n!mv test/test test/data","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:14:08.521897Z","iopub.execute_input":"2023-06-11T01:14:08.522645Z","iopub.status.idle":"2023-06-11T01:14:10.631612Z","shell.execute_reply.started":"2023-06-11T01:14:08.522612Z","shell.execute_reply":"2023-06-11T01:14:10.630254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extrair nome das imagens de teste","metadata":{}},{"cell_type":"code","source":"from os import walk\ntest_path = '/kaggle/input/dog-breed-identification/test/'\ntest_filenames = next(walk(test_path), (None, None, []))[2]","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:14:10.634325Z","iopub.execute_input":"2023-06-11T01:14:10.635123Z","iopub.status.idle":"2023-06-11T01:14:12.896959Z","shell.execute_reply.started":"2023-06-11T01:14:10.635083Z","shell.execute_reply":"2023-06-11T01:14:12.895918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preparar data generator\nPara evitar estouro no consumo de memória","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntest_data_gen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_data_gen.flow_from_directory(\n    directory='test',\n    target_size=(IMG_SIZE, IMG_SIZE),\n    batch_size=BATCH_SIZE,\n    class_mode=None,\n    shuffle=False,\n    seed=42\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:14:12.898773Z","iopub.execute_input":"2023-06-11T01:14:12.899574Z","iopub.status.idle":"2023-06-11T01:14:13.242819Z","shell.execute_reply.started":"2023-06-11T01:14:12.899528Z","shell.execute_reply":"2023-06-11T01:14:13.241889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images_ids = [image_id.split('.')[0] for image_id in test_filenames]\nprobabilities = model.predict(test_generator)\npredictions = np.argmax(probabilities, axis=-1)\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:14:13.244223Z","iopub.execute_input":"2023-06-11T01:14:13.244623Z","iopub.status.idle":"2023-06-11T01:14:47.022126Z","shell.execute_reply.started":"2023-06-11T01:14:13.244586Z","shell.execute_reply":"2023-06-11T01:14:47.020876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[format(z, '.4f') for z in list(probabilities[0])]","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:14:47.023708Z","iopub.execute_input":"2023-06-11T01:14:47.024192Z","iopub.status.idle":"2023-06-11T01:14:47.035509Z","shell.execute_reply.started":"2023-06-11T01:14:47.024159Z","shell.execute_reply":"2023-06-11T01:14:47.034367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparar dados p/ gerar csv de saída","metadata":{}},{"cell_type":"code","source":"# Preparar dicionário p/ instanciar dataframe\ndict_results = {'id': []}\nbreeds = df.breed.unique()\n\nfor breed in breeds:\n    dict_results[breed] = []\n\ndict_results","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:29:07.617812Z","iopub.execute_input":"2023-06-11T01:29:07.618303Z","iopub.status.idle":"2023-06-11T01:29:07.634973Z","shell.execute_reply.started":"2023-06-11T01:29:07.618260Z","shell.execute_reply":"2023-06-11T01:29:07.633835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (image_id, probabilities_instance) in zip(test_images_ids, probabilities):\n    dict_results['id'].append(image_id)\n    \n    for breed_index, breed in enumerate(breeds):\n        dict_results[breed].append(format(probabilities_instance[breed_index], '.4f'))","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:29:11.381250Z","iopub.execute_input":"2023-06-11T01:29:11.381613Z","iopub.status.idle":"2023-06-11T01:29:13.352169Z","shell.execute_reply.started":"2023-06-11T01:29:11.381583Z","shell.execute_reply":"2023-06-11T01:29:13.351116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_dataframe = pd.DataFrame(data=dict_results)\noutput_dataframe.to_csv('submission.csv', index=False, sep=',')","metadata":{"execution":{"iopub.status.busy":"2023-06-11T01:29:15.040346Z","iopub.execute_input":"2023-06-11T01:29:15.040730Z","iopub.status.idle":"2023-06-11T01:29:16.036842Z","shell.execute_reply.started":"2023-06-11T01:29:15.040699Z","shell.execute_reply":"2023-06-11T01:29:16.035691Z"},"trusted":true},"execution_count":null,"outputs":[]}]}