{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Dogs vs Cats","metadata":{"id":"jOe0O8tZIgAD"}},{"cell_type":"markdown","source":"https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition","metadata":{"id":"RrtQDt6ZIgAG"}},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.applications import vgg16\n\nprint(tf.__version__)\nprint(tf.executing_eagerly())","metadata":{"id":"Q9GLX-_RIgAG","executionInfo":{"status":"ok","timestamp":1712672261074,"user_tz":-180,"elapsed":11132,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"1a2695bb-2ae5-47b8-9677-1e367b89b3dc","execution":{"iopub.status.busy":"2024-04-14T13:29:05.650181Z","iopub.execute_input":"2024-04-14T13:29:05.650609Z","iopub.status.idle":"2024-04-14T13:29:20.795761Z","shell.execute_reply.started":"2024-04-14T13:29:05.650575Z","shell.execute_reply":"2024-04-14T13:29:20.794541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from zipfile import ZipFile\n\nwith ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('') \nwith ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T13:39:13.201118Z","iopub.execute_input":"2024-04-14T13:39:13.202085Z","iopub.status.idle":"2024-04-14T13:39:32.282947Z","shell.execute_reply.started":"2024-04-14T13:39:13.202046Z","shell.execute_reply":"2024-04-14T13:39:32.281767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom random import shuffle\nfrom glob import glob\n\nIMG_SIZE = (224, 224)  # размер входного изображения сети\n\ntrain_dir = 'train'\ntest_dir = 'test'\ntrain_files = glob(os.path.join(train_dir,'*.jpg'))\ntest_files = glob(os.path.join(test_dir, '*.jpg'))\n\n# загружаем входное изображение и предобрабатываем\ndef load_image(path, target_size=IMG_SIZE):\n    img = cv2.imread(path)[...,::-1]\n    img = cv2.resize(img, target_size)\n    return vgg16.preprocess_input(img)  # предобработка для VGG16\n\n# функция-генератор загрузки обучающих данных с диска\ndef fit_generator(files, batch_size=32):\n    batch_size = min(batch_size, len(files))\n    while True:\n        shuffle(files)\n        for k in range(len(files) // batch_size):\n            i = k * batch_size\n            j = i + batch_size\n            if j > len(files):\n                j = - j % len(files)\n            x = np.array([load_image(path) for path in files[i:j]])\n            y = np.array([1. if os.path.basename(path).startswith('dog') else 0.\n                          for path in files[i:j]])\n            yield (x, y)\n\n# функция-генератор загрузки тестовых изображений с диска\ndef predict_generator(files):\n    while True:\n        for path in files:\n            yield np.array([load_image(path)])","metadata":{"execution":{"iopub.status.busy":"2024-04-14T13:41:27.319234Z","iopub.execute_input":"2024-04-14T13:41:27.319694Z","iopub.status.idle":"2024-04-14T13:41:27.493692Z","shell.execute_reply.started":"2024-04-14T13:41:27.319662Z","shell.execute_reply":"2024-04-14T13:41:27.492121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Функции загрузки данных","metadata":{"id":"heSnXQK1IgAI"}},{"cell_type":"code","source":"import os\nfrom random import shuffle\n\n\nIMG_SIZE = (224, 224)  # размер входного изображения сети\n\n\n# загружаем входное изображение и предобрабатываем\ndef load_image(path, target_size=IMG_SIZE):\n    img = cv2.imread(path)[...,::-1]\n    img = cv2.resize(img, target_size)\n    return vgg16.preprocess_input(img)  # предобработка для VGG16\n\n# функция-генератор загрузки обучающих данных с диска\ndef fit_generator(files, batch_size=32):\n    batch_size = min(batch_size, len(files))\n    while True:\n        shuffle(files)\n        for k in range(len(files) // batch_size):\n            i = k * batch_size\n            j = i + batch_size\n            if j > len(files):\n                j = - j % len(files)\n            x = np.array([load_image(path) for path in files[i:j]])\n            y = np.array([1. if os.path.basename(path).startswith('dog') else 0.\n                          for path in files[i:j]])\n            yield (x, y)\n\n# функция-генератор загрузки тестовых изображений с диска\ndef predict_generator(files):\n    while True:\n        for path in files:\n            yield np.array([load_image(path)])","metadata":{"id":"fNFWHrMcIgAI","executionInfo":{"status":"ok","timestamp":1712672270943,"user_tz":-180,"elapsed":405,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"execution":{"iopub.status.busy":"2024-04-14T09:52:34.029391Z","iopub.execute_input":"2024-04-14T09:52:34.029812Z","iopub.status.idle":"2024-04-14T09:52:34.039053Z","shell.execute_reply.started":"2024-04-14T09:52:34.02978Z","shell.execute_reply":"2024-04-14T09:52:34.038118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Визуализируем примеры для обучения","metadata":{"id":"GB1igEWoIgAI"}},{"cell_type":"code","source":"%matplotlib inline\nfrom matplotlib import pyplot as plt\nfig = plt.figure(figsize=(16, 8))\nfor i, path in enumerate(train_files[:10], 1):\n    subplot = fig.add_subplot(2, 5, i)\n    subplot.set_title('%s' % path.split('/')[-1])\n    img = cv2.imread(path)[...,::-1]\n    img = cv2.resize(img, IMG_SIZE)\n    plt.imshow(img)","metadata":{"id":"zkV2eKByIgAJ","executionInfo":{"status":"ok","timestamp":1712672278804,"user_tz":-180,"elapsed":5512,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"50403691-57df-478b-b963-658471486b53","execution":{"iopub.status.busy":"2024-04-14T13:42:20.068966Z","iopub.execute_input":"2024-04-14T13:42:20.069401Z","iopub.status.idle":"2024-04-14T13:42:22.663888Z","shell.execute_reply.started":"2024-04-14T13:42:20.069367Z","shell.execute_reply":"2024-04-14T13:42:22.660976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Загружаем предобученную модель","metadata":{"id":"l8hTEcqxIgAJ"}},{"cell_type":"code","source":"# base_model - объект класса keras.models.Model (Functional Model)\nbase_model = vgg16.VGG16(weights='imagenet',\n                         include_top=False,\n                         input_shape=(IMG_SIZE[0], IMG_SIZE[1], 3))","metadata":{"id":"vXRVzJg5IgAK","executionInfo":{"status":"ok","timestamp":1712672312819,"user_tz":-180,"elapsed":863,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"execution":{"iopub.status.busy":"2024-04-14T13:42:30.314761Z","iopub.execute_input":"2024-04-14T13:42:30.315348Z","iopub.status.idle":"2024-04-14T13:42:31.36116Z","shell.execute_reply.started":"2024-04-14T13:42:30.315304Z","shell.execute_reply":"2024-04-14T13:42:31.36Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model.summary()","metadata":{"id":"9suWBCqmIgAK","executionInfo":{"status":"ok","timestamp":1712672316584,"user_tz":-180,"elapsed":350,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"e3919192-6e59-4281-abc2-1e1517405c48","execution":{"iopub.status.busy":"2024-04-14T13:42:36.099004Z","iopub.execute_input":"2024-04-14T13:42:36.099405Z","iopub.status.idle":"2024-04-14T13:42:36.149523Z","shell.execute_reply.started":"2024-04-14T13:42:36.099373Z","shell.execute_reply":"2024-04-14T13:42:36.148305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Добавляем полносвязный слой","metadata":{"id":"Fx4r0jwmIgAK"}},{"cell_type":"code","source":"# фиксируем все веса предобученной сети\nfor layer in base_model.layers:\n    layer.trainable = False\n\nx = base_model.layers[-5].output\nx = tf.keras.layers.Flatten()(x)\nx = tf.keras.layers.Dense(1,  # один выход (бинарная классификация)\n                          activation='sigmoid',  # функция активации\n                          kernel_regularizer=tf.keras.regularizers.l1(1e-4))(x)\n\nmodel = tf.keras.Model(inputs=base_model.input, outputs=x, name='dogs_vs_cats')","metadata":{"id":"LTs9bbgqIgAL","executionInfo":{"status":"ok","timestamp":1712672324902,"user_tz":-180,"elapsed":380,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"execution":{"iopub.status.busy":"2024-04-14T13:42:43.114267Z","iopub.execute_input":"2024-04-14T13:42:43.114902Z","iopub.status.idle":"2024-04-14T13:42:43.144243Z","shell.execute_reply.started":"2024-04-14T13:42:43.114871Z","shell.execute_reply":"2024-04-14T13:42:43.142908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Выводим архитектуру модели","metadata":{"id":"ef1ELOYIIgAL"}},{"cell_type":"code","source":"model.summary()","metadata":{"id":"dQ2I5V2hIgAL","executionInfo":{"status":"ok","timestamp":1712672328747,"user_tz":-180,"elapsed":361,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"b9a2b2a6-a057-4ffc-a402-ebc487d39bfe","execution":{"iopub.status.busy":"2024-04-14T13:42:46.319478Z","iopub.execute_input":"2024-04-14T13:42:46.319943Z","iopub.status.idle":"2024-04-14T13:42:46.359043Z","shell.execute_reply.started":"2024-04-14T13:42:46.319912Z","shell.execute_reply":"2024-04-14T13:42:46.357839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Компилируем модель и запускаем обучение","metadata":{"id":"KigbQ3CyIgAL"}},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss='binary_crossentropy',  # функция потерь binary_crossentropy (log loss\n              metrics=['accuracy'])","metadata":{"id":"2iKTDFtbIgAL","executionInfo":{"status":"ok","timestamp":1712672331808,"user_tz":-180,"elapsed":434,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"execution":{"iopub.status.busy":"2024-04-14T13:42:50.017559Z","iopub.execute_input":"2024-04-14T13:42:50.017958Z","iopub.status.idle":"2024-04-14T13:42:50.034761Z","shell.execute_reply.started":"2024-04-14T13:42:50.017927Z","shell.execute_reply":"2024-04-14T13:42:50.032757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_samples = 5  # число изображений в валидационной выборке\n\nshuffle(train_files)  # перемешиваем обучающую выборку\nvalidation_data = next(fit_generator(train_files[:val_samples], val_samples))\ntrain_data = fit_generator(train_files[val_samples:])  # данные читаем функцией-генератором\n\n# запускаем процесс обучения\nmodel.fit(train_data,\n          steps_per_epoch=10,  # число вызовов генератора за эпоху\n          epochs=10,  # число эпох обучения\n          validation_data=validation_data)","metadata":{"id":"aOOc8FpJIgAL","executionInfo":{"status":"ok","timestamp":1712674231628,"user_tz":-180,"elapsed":1883937,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"c01adcd9-7583-4de1-d04b-ae71aa6b36ca","execution":{"iopub.status.busy":"2024-04-14T13:43:05.345728Z","iopub.execute_input":"2024-04-14T13:43:05.346136Z","iopub.status.idle":"2024-04-14T13:57:28.978275Z","shell.execute_reply.started":"2024-04-14T13:43:05.346108Z","shell.execute_reply":"2024-04-14T13:57:28.976061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('cats-dogs-vgg16.hdf5')","metadata":{"id":"zru_EGK9IgAM","executionInfo":{"status":"ok","timestamp":1712674234576,"user_tz":-180,"elapsed":363,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"a852e724-fd38-42fb-8fcd-386cc435525d","execution":{"iopub.status.busy":"2024-04-14T10:02:53.061227Z","iopub.execute_input":"2024-04-14T10:02:53.061863Z","iopub.status.idle":"2024-04-14T10:02:53.163762Z","shell.execute_reply.started":"2024-04-14T10:02:53.061829Z","shell.execute_reply":"2024-04-14T10:02:53.162041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Предсказания на проверочной выборке","metadata":{"id":"PicG7uvFIgAM"}},{"cell_type":"code","source":"test_pred = model.predict(\n    predict_generator(train_files), steps=len(train_files))","metadata":{"id":"E2DenGNxIgAM","executionInfo":{"status":"ok","timestamp":1712674244419,"user_tz":-180,"elapsed":7244,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"3f8cc436-6d45-4efb-c15e-24596abb35fa","execution":{"iopub.status.busy":"2024-04-14T13:59:30.968992Z","iopub.execute_input":"2024-04-14T13:59:30.969634Z","iopub.status.idle":"2024-04-14T13:59:31.130157Z","shell.execute_reply.started":"2024-04-14T13:59:30.969589Z","shell.execute_reply":"2024-04-14T13:59:31.128211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(16, 8))\nfor i, (path, score) in enumerate(zip(test_files[:10], test_pred[:10]), 1):\n    subplot = fig.add_subplot(2, 5, i)\n    subplot.set_title('%.2f %s' % (score, os.path.basename(path)))\n    img = cv2.imread(path)[...,::-1]\n    img = cv2.resize(img, IMG_SIZE)\n    subplot.imshow(img)","metadata":{"id":"vugNMskPIgAM","executionInfo":{"status":"ok","timestamp":1712674256970,"user_tz":-180,"elapsed":6698,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"bd9e9433-0774-47c1-b9e6-493985d13e6c","execution":{"iopub.status.busy":"2024-04-14T14:00:15.640505Z","iopub.execute_input":"2024-04-14T14:00:15.641021Z","iopub.status.idle":"2024-04-14T14:00:15.714889Z","shell.execute_reply.started":"2024-04-14T14:00:15.640985Z","shell.execute_reply":"2024-04-14T14:00:15.712819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import log_loss\n\n# вычисляем метрику Log Loss на валидации\nlogloss = log_loss(validation_data[1], model.predict(validation_data[0]))\nprint('Log Loss:', logloss)","metadata":{"id":"TxNcL_VeV43x","executionInfo":{"status":"ok","timestamp":1712674320105,"user_tz":-180,"elapsed":8054,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"cfeff9a4-f4f0-494e-8094-7d6e11601aa9","execution":{"iopub.status.busy":"2024-04-14T14:01:21.933314Z","iopub.execute_input":"2024-04-14T14:01:21.934866Z","iopub.status.idle":"2024-04-14T14:01:25.185239Z","shell.execute_reply.started":"2024-04-14T14:01:21.934821Z","shell.execute_reply":"2024-04-14T14:01:25.183534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Готовим данные для отправки","metadata":{"id":"yhO_4vreIgAM"}},{"cell_type":"code","source":"import re\n\nwith open('submit.txt', 'w') as dst:\n    dst.write('id,label\\n')\n    for path, score in zip(test_files, test_pred):\n        dst.write('%s,%f\\n' % (re.search('(\\d+).jpg$', path).group(1), score))","metadata":{"id":"JhWvLsxkIgAM","executionInfo":{"status":"ok","timestamp":1712674327354,"user_tz":-180,"elapsed":347,"user":{"displayName":"Viktor Moskovchenko","userId":"12791894187146523481"}},"outputId":"04b93937-5012-491a-d509-c89c4c3adab5","execution":{"iopub.status.busy":"2024-04-14T10:02:53.29037Z","iopub.status.idle":"2024-04-14T10:02:53.290722Z","shell.execute_reply.started":"2024-04-14T10:02:53.290576Z","shell.execute_reply":"2024-04-14T10:02:53.29059Z"},"trusted":true},"execution_count":null,"outputs":[]}]}