{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"},{"sourceId":13406,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":11099}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-03T04:28:18.760632Z","iopub.execute_input":"2024-03-03T04:28:18.762086Z","iopub.status.idle":"2024-03-03T04:28:20.024913Z","shell.execute_reply.started":"2024-03-03T04:28:18.762035Z","shell.execute_reply":"2024-03-03T04:28:20.023333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport glob\nimport zipfile\nimport tensorflow as tf\nfrom tensorflow.keras.applications import vgg16\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom random import shuffle\nfrom sklearn.model_selection import train_test_split\n\nprint(tf.__version__)\nprint(tf.executing_eagerly())","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:28:21.483487Z","iopub.execute_input":"2024-03-03T04:28:21.484096Z","iopub.status.idle":"2024-03-03T04:28:31.999975Z","shell.execute_reply.started":"2024-03-03T04:28:21.484061Z","shell.execute_reply":"2024-03-03T04:28:31.998645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = 'train'\ntest_dir = 'test'\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('')\n    \nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('')\ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:28:32.002582Z","iopub.execute_input":"2024-03-03T04:28:32.003511Z","iopub.status.idle":"2024-03-03T04:28:51.955831Z","shell.execute_reply.started":"2024-03-03T04:28:32.003468Z","shell.execute_reply":"2024-03-03T04:28:51.954492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split('/')[-1].split('.')[0] for path in train_list]\nprint(f\"Train Data: {len(labels)}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:32.976917Z","iopub.execute_input":"2024-03-02T18:02:32.977398Z","iopub.status.idle":"2024-03-02T18:02:33.009463Z","shell.execute_reply.started":"2024-03-02T18:02:32.977364Z","shell.execute_reply":"2024-03-02T18:02:33.006886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=10)\nfig, axes = plt.subplots(2, 5, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:33.012055Z","iopub.execute_input":"2024-03-02T18:02:33.012517Z","iopub.status.idle":"2024-03-02T18:02:35.870956Z","shell.execute_reply.started":"2024-03-02T18:02:33.012483Z","shell.execute_reply":"2024-03-02T18:02:35.868759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = (224, 224)  # размер входного изображения сети\n\n# # загружаем входное изображение и предобрабатываем\n# def load_image(path, target_size=IMG_SIZE):\n#     for i in path:\n#         img = cv2.imread(i)[...,::-1]\n#         img = cv2.resize(img, target_size)\n#         return vgg16.preprocess_input(img)  # предобработка для VGG16\n\n# функция-генератор загрузки обучающих данных с диска\ndef fit_generator(files, batch_size=32):\n    batch_size = min(batch_size, len(files))\n    while True:\n        shuffle(files)\n        for k in range(len(files) // batch_size):\n            i = k * batch_size\n            j = i + batch_size\n            if j > len(files):\n                j = - j % len(files)\n            x = np.array([load_image(path) for path in files[i:j]])\n            y = np.array([1. if os.path.basename(path).startswith('dog') else 0.\n                          for path in files[i:j]])\n            yield (x, y)\n\n# функция-генератор загрузки тестовых изображений с диска\n# def predict_generator(files):\n#     while True:\n#         for path in files:\n#             yield np.array([load_image(path)])","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:35.874158Z","iopub.execute_input":"2024-03-02T18:02:35.876122Z","iopub.status.idle":"2024-03-02T18:02:35.895333Z","shell.execute_reply.started":"2024-03-02T18:02:35.876046Z","shell.execute_reply":"2024-03-02T18:02:35.892966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base_model - объект класса keras.models.Model (Functional Model)\nbase_model = vgg16.VGG16(weights='imagenet',\n                         include_top=False,\n                         input_shape=(IMG_SIZE[0], IMG_SIZE[1], 3))","metadata":{"execution":{"iopub.status.busy":"2024-03-02T08:29:54.390151Z","iopub.execute_input":"2024-03-02T08:29:54.390614Z","iopub.status.idle":"2024-03-02T08:29:54.811059Z","shell.execute_reply.started":"2024-03-02T08:29:54.390572Z","shell.execute_reply":"2024-03-02T08:29:54.809712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:36.423655Z","iopub.execute_input":"2024-03-02T18:02:36.424191Z","iopub.status.idle":"2024-03-02T18:02:36.483615Z","shell.execute_reply.started":"2024-03-02T18:02:36.424154Z","shell.execute_reply":"2024-03-02T18:02:36.482186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# фиксируем все веса предобученной сети\nfor layer in base_model.layers:\n    layer.trainable = False\n\nx = base_model.layers[-5].output\nx = tf.keras.layers.Flatten()(x)\nx = tf.keras.layers.Dense(1,  # один выход (бинарная классификация)\n                          activation='sigmoid',  # функция активации  \n                          kernel_regularizer=tf.keras.regularizers.l1(1e-4))(x)\n\nmodel = tf.keras.Model(inputs=base_model.input, outputs=x, name='dogs_vs_cats')","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:42.019451Z","iopub.execute_input":"2024-03-02T18:02:42.019978Z","iopub.status.idle":"2024-03-02T18:02:42.053603Z","shell.execute_reply.started":"2024-03-02T18:02:42.019941Z","shell.execute_reply":"2024-03-02T18:02:42.051986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:46.111751Z","iopub.execute_input":"2024-03-02T18:02:46.112225Z","iopub.status.idle":"2024-03-02T18:02:46.154635Z","shell.execute_reply.started":"2024-03-02T18:02:46.11219Z","shell.execute_reply":"2024-03-02T18:02:46.153316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', \n              loss='binary_crossentropy',  # функция потерь binary_crossentropy (log loss\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-03-02T18:02:51.043724Z","iopub.execute_input":"2024-03-02T18:02:51.044271Z","iopub.status.idle":"2024-03-02T18:02:51.057316Z","shell.execute_reply.started":"2024-03-02T18:02:51.044233Z","shell.execute_reply":"2024-03-02T18:02:51.055776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_samples = 10  # число изображений в валидационной выборке\n\nshuffle(train_list)  # перемешиваем обучающую выборку\nvalidation_data = next(fit_generator(train_list[:val_samples], val_samples))\ntrain_data = fit_generator(train_list[val_samples:])  # данные читаем функцией-генератором\n\n# запускаем процесс обучения\nmodel.fit(train_data,\n          steps_per_epoch=10,  # число вызовов генератора за эпоху\n          epochs=100,  # число эпох обучения\n          validation_data=validation_data)","metadata":{"execution":{"iopub.status.busy":"2024-03-02T08:31:15.720813Z","iopub.execute_input":"2024-03-02T08:31:15.721336Z","iopub.status.idle":"2024-03-02T11:00:39.318308Z","shell.execute_reply.started":"2024-03-02T08:31:15.721299Z","shell.execute_reply":"2024-03-02T11:00:39.314317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('cats-dogs-vgg16.hdf5')","metadata":{"execution":{"iopub.status.busy":"2024-03-02T11:02:05.896804Z","iopub.execute_input":"2024-03-02T11:02:05.897445Z","iopub.status.idle":"2024-03-02T11:02:06.086764Z","shell.execute_reply.started":"2024-03-02T11:02:05.897334Z","shell.execute_reply":"2024-03-02T11:02:06.085102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:28:51.958541Z","iopub.execute_input":"2024-03-03T04:28:51.959449Z","iopub.status.idle":"2024-03-03T04:28:51.965259Z","shell.execute_reply.started":"2024-03-03T04:28:51.959401Z","shell.execute_reply":"2024-03-03T04:28:51.963893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_model('/kaggle/input/cats-dogs-vgg16/keras/1/1/cats-dogs-vgg16.hdf5')","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:28:52.591855Z","iopub.execute_input":"2024-03-03T04:28:52.592226Z","iopub.status.idle":"2024-03-03T04:28:53.027638Z","shell.execute_reply.started":"2024-03-03T04:28:52.592196Z","shell.execute_reply":"2024-03-03T04:28:53.026451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = (224, 224)\ndef load_image1(path, target_size=IMG_SIZE):\n    img_list = []\n    for i in range(len(path)):  # Проходим по индексам списка path\n        img = cv2.imread(path[i])[...,::-1]\n        img = cv2.resize(img, target_size)\n        img_list.append(vgg16.preprocess_input(img))\n    return img_list","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:28:53.029217Z","iopub.execute_input":"2024-03-03T04:28:53.029652Z","iopub.status.idle":"2024-03-03T04:28:53.037369Z","shell.execute_reply.started":"2024-03-03T04:28:53.029612Z","shell.execute_reply":"2024-03-03T04:28:53.036272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_load = load_image1(test_list)\ngenerator_data = np.array(image_load)","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:28:53.038675Z","iopub.execute_input":"2024-03-03T04:28:53.039042Z","iopub.status.idle":"2024-03-03T04:30:03.20446Z","shell.execute_reply.started":"2024-03-03T04:28:53.039013Z","shell.execute_reply":"2024-03-03T04:30:03.202848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = model.predict(generator_data, steps=len(test_list))","metadata":{"execution":{"iopub.status.busy":"2024-03-03T04:30:03.207775Z","iopub.execute_input":"2024-03-03T04:30:03.208316Z","iopub.status.idle":"2024-03-03T05:17:40.529665Z","shell.execute_reply.started":"2024-03-03T04:30:03.20826Z","shell.execute_reply":"2024-03-03T05:17:40.527871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(16, 8))\nfor i, (path, score) in enumerate(zip(test_list[:10], test_pred[:10]), 1):\n    subplot = fig.add_subplot(2, 5, i)\n    subplot.set_title('%.2f %s' % (score, os.path.basename(path)))\n    img = cv2.imread(path)[...,::-1]\n    img = cv2.resize(img, IMG_SIZE)\n    subplot.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-03-03T05:17:40.532456Z","iopub.execute_input":"2024-03-03T05:17:40.532867Z","iopub.status.idle":"2024-03-03T05:17:43.041528Z","shell.execute_reply.started":"2024-03-03T05:17:40.532833Z","shell.execute_reply":"2024-03-03T05:17:43.040193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\nwith open('submit.txt', 'w') as dst:\n    dst.write('id,label\\n')\n    for path, score in zip(test_list, test_pred):\n        dst.write('%s,%f\\n' % (re.search('(\\d+).jpg$', path).group(1), score))","metadata":{"execution":{"iopub.status.busy":"2024-03-03T05:20:31.044161Z","iopub.execute_input":"2024-03-03T05:20:31.044675Z","iopub.status.idle":"2024-03-03T05:20:31.09178Z","shell.execute_reply.started":"2024-03-03T05:20:31.044641Z","shell.execute_reply":"2024-03-03T05:20:31.090796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\nwith open('submit1.txt', 'w') as dst:\n    dst.write('id,label\\n')\n    for path, score in zip(test_list, test_pred):\n        image_id = re.search('(\\d+).jpg$', path).group(1)\n        dst.write('%s,%f\\n' % (image_id, score))","metadata":{"execution":{"iopub.status.busy":"2024-03-03T05:20:38.976829Z","iopub.execute_input":"2024-03-03T05:20:38.977735Z","iopub.status.idle":"2024-03-03T05:20:39.029142Z","shell.execute_reply.started":"2024-03-03T05:20:38.977666Z","shell.execute_reply":"2024-03-03T05:20:39.02768Z"},"trusted":true},"execution_count":null,"outputs":[]}]}