{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"В этом кагле получилось супер очевидное решение к которому мы пришли очень трудным и долгим путем. \nНачали с того что попробовали аугментировать изображения (код созранен но закомментировать ниже рабочего пайплайна), использовали рескейлы, повороты, зумы, отражения. Далее начали выбирать модель, попробвали написать несколько CNN сами и потюнить их, далее перешли к использованию собраных моделей resnet mobilenet efficientnet, однако продолжали получать плохие скоры. Даже попробовали использовать результаты Yolo как фичи для cnn, а потом и вовсе хотели использовать только его предсказания. В итоге нашли проблему в генераторах которыми мы передвалаи изображения для обучения. Стали использовать стандарную cv2.read_image и не передавать изображения через генератор, из-за этого стали биться в предел рама в kaggle kernel, пришлось использовать mobilenetv2. в итоге получили log_loss = 0.08873","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:15:39.226098Z","iopub.execute_input":"2025-03-18T20:15:39.226538Z","iopub.status.idle":"2025-03-18T20:15:39.237795Z","shell.execute_reply.started":"2025-03-18T20:15:39.226508Z","shell.execute_reply":"2025-03-18T20:15:39.236994Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv\n/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip\n/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip\n","output_type":"stream"}],"execution_count":29},{"cell_type":"code","source":"import zipfile\nimport os\n\n# Пути к архивам\ntrain_zip_path = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\ntest_zip_path = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\n\n# Папки для распаковки\ntrain_extract_path = '/kaggle/working/train_images'\ntest_extract_path = '/kaggle/working/test_images'\n\n# Создаем папки, если их нет\nos.makedirs(train_extract_path, exist_ok=True)\nos.makedirs(test_extract_path, exist_ok=True)\n\n# Распаковка train.zip\nwith zipfile.ZipFile(train_zip_path, 'r') as zip_ref:\n zip_ref.extractall(train_extract_path)\nprint(\"Train extraction sample:\", os.listdir(train_extract_path)[:10])\n\n# Распаковка test.zip\nwith zipfile.ZipFile(test_zip_path, 'r') as zip_ref:\n zip_ref.extractall(test_extract_path)\nprint(\"Test extraction sample:\", os.listdir(test_extract_path)[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:15:39.640778Z","iopub.execute_input":"2025-03-18T20:15:39.641028Z","iopub.status.idle":"2025-03-18T20:15:49.787612Z","shell.execute_reply.started":"2025-03-18T20:15:39.641008Z","shell.execute_reply":"2025-03-18T20:15:49.786642Z"}},"outputs":[{"name":"stdout","text":"Train extraction sample: ['train']\nTest extraction sample: ['test']\n","output_type":"stream"}],"execution_count":30},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.config.list_physical_devices('GPU'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:15:49.788884Z","iopub.execute_input":"2025-03-18T20:15:49.789252Z","iopub.status.idle":"2025-03-18T20:15:49.793243Z","shell.execute_reply.started":"2025-03-18T20:15:49.789213Z","shell.execute_reply":"2025-03-18T20:15:49.792507Z"}},"outputs":[{"name":"stdout","text":"[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU')]\n","output_type":"stream"}],"execution_count":31},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import EarlyStopping\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:15:49.794907Z","iopub.execute_input":"2025-03-18T20:15:49.795121Z","iopub.status.idle":"2025-03-18T20:15:49.81072Z","shell.execute_reply.started":"2025-03-18T20:15:49.795091Z","shell.execute_reply":"2025-03-18T20:15:49.809861Z"}},"outputs":[],"execution_count":32},{"cell_type":"code","source":"train_dir = '/kaggle/working/train_images/train'\ntest_dir = '/kaggle/working/test_images/test'\n\n# Получаем список файлов в train-папке\ntrain_files = os.listdir(train_dir)\nprint(\"Общее количество изображений в train:\", len(train_files))\n\n# Фильтруем файлы: оставляем только те, что начинаются с 'cat' или 'dog'\ntrain_files = [f for f in train_files if f.startswith('cat') or f.startswith('dog')]\nprint(\"Количество изображений после фильтрации:\", len(train_files))\n\n# Формируем DataFrame с метками и полными путями\ndf = pd.DataFrame({'filename': train_files})\ndf['label'] = df['filename'].apply(lambda x: 'cat' if x.startswith('cat') else 'dog')\ndf['filepath'] = df['filename'].apply(lambda x: os.path.join(train_dir, x))\n\n# Выбираем по 1500 изображений для каждого класса\ncats_df = df[df['label'] == 'cat'].sample(n=12500, random_state=42)\ndogs_df = df[df['label'] == 'dog'].sample(n=12500, random_state=42)\ndf_subset = pd.concat([cats_df, dogs_df]).reset_index(drop=True)\nprint(\"Используемый датасет (1500 кошек и 1500 собак):\", df_subset.shape)\n\n# Разбиваем на обучающую и валидационную выборки (80/20)\ntrain_df, val_df = train_test_split(df_subset, test_size=0.2, stratify=df_subset['label'], random_state=42)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:15:49.811856Z","iopub.execute_input":"2025-03-18T20:15:49.8121Z","iopub.status.idle":"2025-03-18T20:15:49.907202Z","shell.execute_reply.started":"2025-03-18T20:15:49.812079Z","shell.execute_reply":"2025-03-18T20:15:49.906368Z"}},"outputs":[{"name":"stdout","text":"Общее количество изображений в train: 25000\nКоличество изображений после фильтрации: 25000\nИспользуемый датасет (1500 кошек и 1500 собак): (25000, 3)\n","output_type":"stream"}],"execution_count":33},{"cell_type":"code","source":"def apply_clahe(image):\n \"\"\"\n Применяет CLAHE к каналу яркости изображения в цветовом пространстве LAB.\n image: изображение в формате RGB.\n Возвращает изображение в формате RGB с улучшенным контрастом.\n \"\"\"\n lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)\n l, a, b = cv2.split(lab)\n clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n cl = clahe.apply(l)\n limg = cv2.merge((cl, a, b))\n return cv2.cvtColor(limg, cv2.COLOR_LAB2RGB)\n\ndef load_images_and_labels(df, target_size=(160, 160)):\n images = []\n labels = []\n for idx, row in df.iterrows():\n img = cv2.imread(row['filepath'])\n if img is None:\n continue \n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n img = apply_clahe(img)\n img = cv2.resize(img, target_size)\n img = img.astype(\"float32\") / 255.0\n images.append(img)\n label = 0 if row['label'] == 'cat' else 1\n labels.append(label)\n return np.array(images), np.array(labels)\n\ndef load_test_images(df, target_size=(160, 160)):\n images = []\n ids = []\n for idx, row in df.iterrows():\n img = cv2.imread(row['filepath'])\n if img is None:\n continue\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n img = apply_clahe(img)\n img = cv2.resize(img, target_size)\n img = img.astype(\"float32\") / 255.0\n images.append(img)\n ids.append(row['id'])\n return np.array(images), ids\n\n# Задаем размер изображений (160x160 для экономии памяти)\nimg_size = (160, 160)\n\n# Загружаем обучающие и валидационные данные\nX_train, y_train = load_images_and_labels(train_df, target_size=img_size)\nX_val, y_val = load_images_and_labels(val_df, target_size=img_size)\nprint(\"Train shape:\", X_train.shape, y_train.shape)\nprint(\"Validation shape:\", X_val.shape, y_val.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:15:49.908105Z","iopub.execute_input":"2025-03-18T20:15:49.908491Z","iopub.status.idle":"2025-03-18T20:17:43.180548Z","shell.execute_reply.started":"2025-03-18T20:15:49.908458Z","shell.execute_reply":"2025-03-18T20:17:43.179827Z"}},"outputs":[{"name":"stdout","text":"Train shape: (20000, 160, 160, 3) (20000,)\nValidation shape: (5000, 160, 160, 3) (5000,)\n","output_type":"stream"}],"execution_count":34},{"cell_type":"code","source":"base_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(img_size[0], img_size[1], 3))\nbase_model.trainable = False \n\nmodel = models.Sequential([\n base_model,\n layers.GlobalAveragePooling2D(),\n layers.Dense(64, activation='relu'),\n layers.Dropout(0.5),\n layers.Dense(1, activation='sigmoid') \n])\n\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),\n loss='binary_crossentropy',\n metrics=['accuracy'])\n\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:17:43.181318Z","iopub.execute_input":"2025-03-18T20:17:43.181578Z","iopub.status.idle":"2025-03-18T20:17:43.880947Z","shell.execute_reply.started":"2025-03-18T20:17:43.181547Z","shell.execute_reply":"2025-03-18T20:17:43.880134Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"sequential_5\"\u001b[0m\n","text/html":"Model: \"sequential_5\"\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n│ mobilenetv2_1.00_160 (\u001b[38;5;33mFunctional\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5\u001b[0m, \u001b[38;5;34m5\u001b[0m, \u001b[38;5;34m1280\u001b[0m) │ \u001b[38;5;34m2,257,984\u001b[0m │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ global_average_pooling2d_5 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1280\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n│ (\u001b[38;5;33mGlobalAveragePooling2D\u001b[0m) │ │ │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_10 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m) │ \u001b[38;5;34m81,984\u001b[0m │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dropout_5 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_11 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m) │ \u001b[38;5;34m65\u001b[0m │\n└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n","text/html":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n┃ Layer (type) ┃ Output Shape ┃ Param # ┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n│ mobilenetv2_1.00_160 (Functional) │ (None, 5, 5, 1280) │ 2,257,984 │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ global_average_pooling2d_5 │ (None, 1280) │ 0 │\n│ (GlobalAveragePooling2D) │ │ │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_10 (Dense) │ (None, 64) │ 81,984 │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dropout_5 (Dropout) │ (None, 64) │ 0 │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_11 (Dense) │ (None, 1) │ 65 │\n└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,340,033\u001b[0m (8.93 MB)\n","text/html":" Total params: 2,340,033 (8.93 MB)\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m82,049\u001b[0m (320.50 KB)\n","text/html":" Trainable params: 82,049 (320.50 KB)\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m2,257,984\u001b[0m (8.61 MB)\n","text/html":" Non-trainable params: 2,257,984 (8.61 MB)\n
\n"},"metadata":{}}],"execution_count":35},{"cell_type":"code","source":"epochs = 20\nearly_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\n\nhistory = model.fit(\n X_train, y_train,\n validation_data=(X_val, y_val),\n epochs=epochs,\n callbacks=[early_stopping]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:17:43.888929Z","iopub.execute_input":"2025-03-18T20:17:43.889155Z","iopub.status.idle":"2025-03-18T20:20:57.549436Z","shell.execute_reply.started":"2025-03-18T20:17:43.889135Z","shell.execute_reply":"2025-03-18T20:20:57.548591Z"}},"outputs":[{"name":"stdout","text":"Epoch 1/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m29s\u001b[0m 36ms/step - accuracy: 0.8921 - loss: 0.2593 - val_accuracy: 0.9762 - val_loss: 0.0665\nEpoch 2/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9727 - loss: 0.0802 - val_accuracy: 0.9774 - val_loss: 0.0600\nEpoch 3/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9765 - loss: 0.0686 - val_accuracy: 0.9774 - val_loss: 0.0563\nEpoch 4/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 22ms/step - accuracy: 0.9761 - loss: 0.0610 - val_accuracy: 0.9776 - val_loss: 0.0551\nEpoch 5/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9793 - loss: 0.0531 - val_accuracy: 0.9784 - val_loss: 0.0548\nEpoch 6/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9794 - loss: 0.0523 - val_accuracy: 0.9786 - val_loss: 0.0550\nEpoch 7/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9829 - loss: 0.0483 - val_accuracy: 0.9788 - val_loss: 0.0539\nEpoch 8/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9824 - loss: 0.0459 - val_accuracy: 0.9784 - val_loss: 0.0526\nEpoch 9/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9865 - loss: 0.0428 - val_accuracy: 0.9794 - val_loss: 0.0522\nEpoch 10/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 21ms/step - accuracy: 0.9844 - loss: 0.0421 - val_accuracy: 0.9786 - val_loss: 0.0527\nEpoch 11/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 21ms/step - accuracy: 0.9852 - loss: 0.0401 - val_accuracy: 0.9792 - val_loss: 0.0527\nEpoch 12/20\n\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 22ms/step - accuracy: 0.9885 - loss: 0.0358 - val_accuracy: 0.9794 - val_loss: 0.0534\n","output_type":"stream"}],"execution_count":36},{"cell_type":"code","source":"test_files = os.listdir(test_dir)\ntest_files.sort(key=lambda x: int(os.path.splitext(x)[0]))\ntest_df = pd.DataFrame({\n 'filename': test_files,\n 'filepath': [os.path.join(test_dir, f) for f in test_files],\n 'id': [os.path.splitext(f)[0] for f in test_files]\n})\n\nX_test, test_ids = load_test_images(test_df, target_size=img_size)\nprint(\"Test shape:\", X_test.shape)\n\npredictions = model.predict(X_test)\npredicted_probabilities = predictions.flatten()\n\n\nsubmission_df = pd.DataFrame({\n 'id': test_ids,\n 'label': predicted_probabilities\n})\nsubmission_df['id'] = submission_df['id'].astype(int)\nsubmission_df = submission_df.sort_values('id')\n\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"Submission file saved as submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:20:57.558424Z","iopub.execute_input":"2025-03-18T20:20:57.558662Z","iopub.status.idle":"2025-03-18T20:22:12.441647Z","shell.execute_reply.started":"2025-03-18T20:20:57.558636Z","shell.execute_reply":"2025-03-18T20:22:12.440873Z"}},"outputs":[{"name":"stdout","text":"Test shape: (12500, 160, 160, 3)\n\u001b[1m391/391\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 18ms/step\nSubmission file saved as submission.csv\n","output_type":"stream"}],"execution_count":37},{"cell_type":"code","source":"# train_dir = '/kaggle/working/train_images/train'\n# test_dir = '/kaggle/working/test_images/test'\n\n# train_files = os.listdir(train_dir)\n# print('Общее количество изображений в train:', len(train_files))\n\n# cat_files = [f for f in train_files if f.startswith('cat')]\n# dog_files = [f for f in train_files if f.startswith('dog')]\n# print('Количество изображений кошек:', len(cat_files))\n# print('Количество изображений собак:', len(dog_files))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:22:12.442817Z","iopub.execute_input":"2025-03-18T20:22:12.443149Z","iopub.status.idle":"2025-03-18T20:22:12.446631Z","shell.execute_reply.started":"2025-03-18T20:22:12.443114Z","shell.execute_reply":"2025-03-18T20:22:12.445833Z"}},"outputs":[],"execution_count":38},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pandas as pd\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# from PIL import Image\n\n# from sklearn.model_selection import train_test_split\n\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense\n# from tensorflow.keras.optimizers import Adam\n# from tensorflow.keras.preprocessing.image import load_img, img_to_array","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:53.095418Z","iopub.execute_input":"2025-03-18T19:27:53.095707Z","iopub.status.idle":"2025-03-18T19:27:53.100487Z","shell.execute_reply.started":"2025-03-18T19:27:53.095655Z","shell.execute_reply":"2025-03-18T19:27:53.099626Z"}},"outputs":[],"execution_count":70},{"cell_type":"code","source":"# import tensorflow as tf\n# print(tf.config.list_physical_devices('GPU'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:53.101878Z","iopub.execute_input":"2025-03-18T19:27:53.102098Z","iopub.status.idle":"2025-03-18T19:27:54.520023Z","shell.execute_reply.started":"2025-03-18T19:27:53.102079Z","shell.execute_reply":"2025-03-18T19:27:54.519036Z"}},"outputs":[{"name":"stdout","text":"[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU')]\n","output_type":"stream"}],"execution_count":71},{"cell_type":"code","source":"# train_dir = '/kaggle/working/train_images/train'\n# test_dir = '/kaggle/working/test_images/test'\n\n# train_files = os.listdir(train_dir)\n# print('Общее количество изображений в train:', len(train_files))\n\n# cat_files = [f for f in train_files if f.startswith('cat')]\n# dog_files = [f for f in train_files if f.startswith('dog')]\n# print('Количество изображений кошек:', len(cat_files))\n# print('Количество изображений собак:', len(dog_files))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:54.521046Z","iopub.execute_input":"2025-03-18T19:27:54.521397Z","iopub.status.idle":"2025-03-18T19:27:54.551757Z","shell.execute_reply.started":"2025-03-18T19:27:54.521362Z","shell.execute_reply":"2025-03-18T19:27:54.551064Z"}},"outputs":[{"name":"stdout","text":"Общее количество изображений в train: 25000\nКоличество изображений кошек: 12500\nКоличество изображений собак: 12500\n","output_type":"stream"}],"execution_count":72},{"cell_type":"code","source":"# df = pd.DataFrame({'filename': os.listdir(train_dir)})\n# df['label'] = df['filename'].apply(lambda x: 'cat' if x.startswith('cat') else 'dog')\n# df['filepath'] = df['filename'].apply(lambda x: os.path.join(train_dir, x))\n\n# cats_df = df[df['label'] == 'cat'].sample(10000, random_state=42)\n# dogs_df = df[df['label'] == 'dog'].sample(10000, random_state=42)\n\n# df_subset = pd.concat([cats_df, dogs_df]).reset_index(drop=True)\n\n# train_df, val_df = train_test_split(df, test_size=0.2, stratify=df['label'], random_state=42)\n\n# img_width, img_height = 224, 224\n# batch_size = 32","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:54.552492Z","iopub.execute_input":"2025-03-18T19:27:54.552739Z","iopub.status.idle":"2025-03-18T19:27:54.632804Z","shell.execute_reply.started":"2025-03-18T19:27:54.552706Z","shell.execute_reply":"2025-03-18T19:27:54.631851Z"}},"outputs":[],"execution_count":73},{"cell_type":"code","source":"# def display_augmented_images(generator, img_path, num_images=4, title_prefix=\"\"):\n\n# img = load_img(img_path, target_size=(img_width, img_height))\n# x = img_to_array(img)\n# x = x.reshape((1,) + x.shape)\n \n# plt.figure(figsize=(12, 3))\n# for i, batch in enumerate(generator.flow(x, batch_size=1)):\n# plt.subplot(1, num_images, i+1)\n# plt.imshow(batch[0])\n# plt.title(f\"{title_prefix} {i+1}\")\n# plt.axis('off')\n# if i == num_images - 1:\n# break\n# plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:54.633681Z","iopub.execute_input":"2025-03-18T19:27:54.633945Z","iopub.status.idle":"2025-03-18T19:27:54.639409Z","shell.execute_reply.started":"2025-03-18T19:27:54.633925Z","shell.execute_reply":"2025-03-18T19:27:54.638496Z"}},"outputs":[],"execution_count":74},{"cell_type":"code","source":"# sample_img_path = train_df['filepath'].iloc[0]\n\n# original_img = load_img(sample_img_path, target_size=(img_width, img_height))\n# plt.figure(figsize=(3, 3))\n# plt.imshow(original_img)\n# plt.title(\"Оригинальное изображение\")\n# plt.axis('off')\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:54.642256Z","iopub.execute_input":"2025-03-18T19:27:54.642512Z","iopub.status.idle":"2025-03-18T19:27:54.691099Z","shell.execute_reply.started":"2025-03-18T19:27:54.642481Z","shell.execute_reply":"2025-03-18T19:27:54.690323Z"}},"outputs":[],"execution_count":75},{"cell_type":"code","source":"# rescale_gen = ImageDataGenerator(rescale=1./255)\n# display_augmented_images(rescale_gen, sample_img_path, title_prefix=\"Rescaled\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:54.693022Z","iopub.execute_input":"2025-03-18T19:27:54.693271Z","iopub.status.idle":"2025-03-18T19:27:54.840991Z","shell.execute_reply.started":"2025-03-18T19:27:54.693251Z","shell.execute_reply":"2025-03-18T19:27:54.83992Z"}},"outputs":[],"execution_count":76},{"cell_type":"code","source":"# rotation_gen = ImageDataGenerator(rotation_range=40, rescale=1./255)\n# display_augmented_images(rotation_gen, sample_img_path, title_prefix=\"Rotation\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:54.84197Z","iopub.execute_input":"2025-03-18T19:27:54.842281Z","iopub.status.idle":"2025-03-18T19:27:55.024121Z","shell.execute_reply.started":"2025-03-18T19:27:54.842245Z","shell.execute_reply":"2025-03-18T19:27:55.023278Z"}},"outputs":[],"execution_count":77},{"cell_type":"code","source":"# zoom_gen = ImageDataGenerator(zoom_range=0.2, rescale=1./255)\n# display_augmented_images(zoom_gen, sample_img_path, title_prefix=\"Zoom\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.025005Z","iopub.execute_input":"2025-03-18T19:27:55.025322Z","iopub.status.idle":"2025-03-18T19:27:55.209287Z","shell.execute_reply.started":"2025-03-18T19:27:55.025289Z","shell.execute_reply":"2025-03-18T19:27:55.20852Z"}},"outputs":[],"execution_count":78},{"cell_type":"code","source":"# flip_gen = ImageDataGenerator(horizontal_flip=True, rescale=1./255)\n# display_augmented_images(flip_gen, sample_img_path, title_prefix=\"Flip\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.21019Z","iopub.execute_input":"2025-03-18T19:27:55.210537Z","iopub.status.idle":"2025-03-18T19:27:55.361626Z","shell.execute_reply.started":"2025-03-18T19:27:55.2105Z","shell.execute_reply":"2025-03-18T19:27:55.36088Z"}},"outputs":[],"execution_count":79},{"cell_type":"code","source":"# composite_gen = ImageDataGenerator(\n# rescale=1./255,\n# rotation_range=40,\n# zoom_range=0.2,\n# horizontal_flip=True,\n# shear_range=0.2,\n# width_shift_range=0.2,\n# height_shift_range=0.2,\n# fill_mode='nearest'\n# )\n# display_augmented_images(composite_gen, sample_img_path, title_prefix=\"Composite\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.362578Z","iopub.execute_input":"2025-03-18T19:27:55.362951Z","iopub.status.idle":"2025-03-18T19:27:55.546101Z","shell.execute_reply.started":"2025-03-18T19:27:55.362898Z","shell.execute_reply":"2025-03-18T19:27:55.545358Z"}},"outputs":[],"execution_count":80},{"cell_type":"code","source":"# import cv2\n# import numpy as np\n\n# def custom_preprocessing(image):\n# \"\"\"\n# Принимает изображение в формате RGB с пикселями в диапазоне [0, 255].\n# 1. Применяет CLAHE для улучшения контрастности в LAB-пространстве.\n# 2. Вычисляет края с помощью Canny.\n# 3. Объединяет улучшенное изображение и края, сохраняя цветовую информацию.\n# 4. Возвращает трехканальное изображение, нормализованное к диапазону [0, 1].\n# \"\"\"\n# image = np.array(image, dtype=np.uint8)\n \n# lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)\n# l, a, b = cv2.split(lab)\n \n# # Применяем CLAHE к L-каналу\n# clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n# l_clahe = clahe.apply(l)\n \n# # Объединяем каналы обратно\n# lab_clahe = cv2.merge([l_clahe, a, b])\n# enhanced_img = cv2.cvtColor(lab_clahe, cv2.COLOR_LAB2RGB)\n \n# # Вычисляем края из исходного изображения (можно также брать из enhanced_img)\n# gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n# edges = cv2.Canny(gray, threshold1=100, threshold2=200)\n# # Приводим края к трехканальному изображению\n# edges_colored = np.stack((edges,)*3, axis=-1)\n \n\n# blended = cv2.addWeighted(enhanced_img, 0.9, edges_colored, 0.1, 0)\n \n# processed = blended / 255.0\n# return processed\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.54703Z","iopub.execute_input":"2025-03-18T19:27:55.547342Z","iopub.status.idle":"2025-03-18T19:27:55.55161Z","shell.execute_reply.started":"2025-03-18T19:27:55.547315Z","shell.execute_reply":"2025-03-18T19:27:55.55052Z"}},"outputs":[],"execution_count":81},{"cell_type":"code","source":"# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n\n# train_datagen = ImageDataGenerator(\n# rescale=1./255,\n# rotation_range=40,\n# zoom_range=0.2,\n# horizontal_flip=True,\n# shear_range=0.2,\n# width_shift_range=0.2,\n# height_shift_range=0.2,\n# fill_mode='nearest'\n# )\n\n\n# # Генератор для валидационной выборки (без аугментации, только предобработка)\n# val_datagen = ImageDataGenerator(\n# # preprocessing_function=custom_preprocessing\n# )\n\n# train_generator = train_datagen.flow_from_dataframe(\n# train_df,\n# x_col='filepath',\n# y_col='label',\n# target_size=(img_width, img_height),\n# class_mode='binary',\n# batch_size=batch_size\n# )\n\n# validation_generator = val_datagen.flow_from_dataframe(\n# val_df,\n# x_col='filepath',\n# y_col='label',\n# target_size=(img_width, img_height),\n# class_mode='binary',\n# batch_size=batch_size\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.552745Z","iopub.execute_input":"2025-03-18T19:27:55.553086Z","iopub.status.idle":"2025-03-18T19:27:55.567144Z","shell.execute_reply.started":"2025-03-18T19:27:55.553053Z","shell.execute_reply":"2025-03-18T19:27:55.566331Z"}},"outputs":[],"execution_count":82},{"cell_type":"code","source":"# import cv2\n# import numpy as np\n# import pandas as pd\n# import math\n\n# def custom_generator(df, batch_size, target_size, class_mode='binary', augment=False): \n# Аргументы:\n# - df: DataFrame с колонками 'filepath' и 'label'.\n# - batch_size: размер батча.\n# - target_size: кортеж (width, height).\n# - class_mode: 'binary' для бинарной классификации.\n# \"\"\"\n# num_samples = len(df)\n# while True:\n# df = df.sample(frac=1).reset_index(drop=True)\n# for offset in range(0, num_samples, batch_size):\n# batch_df = df.iloc[offset:offset+batch_size]\n# images = []\n# labels = []\n# for _, row in batch_df.iterrows():\n# img = cv2.imread(row['filepath'])\n# if img is None:\n# continue\n# img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n# img = cv2.resize(img, target_size)\n# img = img.astype(\"float32\") / 255.0\n\n# if augment:\n# if np.random.rand() < 0.5:\n# img = cv2.flip(img, 1)\n \n# images.append(img)\n# if class_mode == 'binary':\n# label = 1 if row['label'] == 'dog' else 0\n# labels.append(label)\n# X = np.array(images)\n# y = np.array(labels) if class_mode=='binary' else None\n# yield (X, y)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.567916Z","iopub.execute_input":"2025-03-18T19:27:55.568117Z","iopub.status.idle":"2025-03-18T19:27:55.580056Z","shell.execute_reply.started":"2025-03-18T19:27:55.568089Z","shell.execute_reply":"2025-03-18T19:27:55.579411Z"}},"outputs":[],"execution_count":83},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras.applications import EfficientNetB0\n# from tensorflow.keras import layers, models\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# img_width, img_height = 224, 224\n\n# base_model = EfficientNetB0(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))\n# base_model.trainable = False\n\n# model = models.Sequential([\n# base_model,\n# layers.GlobalAveragePooling2D(),\n# layers.Dense(128, activation='relu'),\n# layers.Dropout(0.5),\n# layers.Dense(1, activation='sigmoid') # Бинарная классификация: собака/кошка\n# ])\n\n# model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),\n# loss='binary_crossentropy',\n# metrics=['accuracy'])\n\n# model.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:27:55.581028Z","iopub.execute_input":"2025-03-18T19:27:55.581258Z","iopub.status.idle":"2025-03-18T19:27:56.570364Z","shell.execute_reply.started":"2025-03-18T19:27:55.581239Z","shell.execute_reply":"2025-03-18T19:27:56.569642Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"sequential_3\"\u001b[0m\n","text/html":"Model: \"sequential_3\"\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n│ efficientnetb0 (\u001b[38;5;33mFunctional\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m1280\u001b[0m) │ \u001b[38;5;34m4,049,571\u001b[0m │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ global_average_pooling2d_3 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1280\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n│ (\u001b[38;5;33mGlobalAveragePooling2D\u001b[0m) │ │ │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_6 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m) │ \u001b[38;5;34m163,968\u001b[0m │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dropout_3 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_7 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m) │ \u001b[38;5;34m129\u001b[0m │\n└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n","text/html":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n┃ Layer (type) ┃ Output Shape ┃ Param # ┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n│ efficientnetb0 (Functional) │ (None, 7, 7, 1280) │ 4,049,571 │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ global_average_pooling2d_3 │ (None, 1280) │ 0 │\n│ (GlobalAveragePooling2D) │ │ │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_6 (Dense) │ (None, 128) │ 163,968 │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dropout_3 (Dropout) │ (None, 128) │ 0 │\n├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n│ dense_7 (Dense) │ (None, 1) │ 129 │\n└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m4,213,668\u001b[0m (16.07 MB)\n","text/html":" Total params: 4,213,668 (16.07 MB)\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m164,097\u001b[0m (641.00 KB)\n","text/html":" Trainable params: 164,097 (641.00 KB)\n
\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m4,049,571\u001b[0m (15.45 MB)\n","text/html":" Non-trainable params: 4,049,571 (15.45 MB)\n
\n"},"metadata":{}}],"execution_count":84},{"cell_type":"code","source":"# from tensorflow.keras.callbacks import EarlyStopping\n\n# early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\n\n# batch_size = 32\n# img_width, img_height = 224, 224 \n# steps_per_epoch = math.ceil(len(train_df) / batch_size)\n# validation_steps = math.ceil(len(val_df) / batch_size)\n\n# train_gen = custom_generator(train_df, batch_size, (img_width, img_height), augment=True)\n# val_gen = custom_generator(val_df, batch_size, (img_width, img_height), augment=False)\n\n# history = model.fit(\n# train_gen,\n# steps_per_epoch=steps_per_epoch,\n# epochs=20,\n# validation_data=val_gen,\n# validation_steps=validation_steps,\n# callbacks=[early_stopping]\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T20:00:02.566842Z","iopub.execute_input":"2025-03-18T20:00:02.56718Z","iopub.status.idle":"2025-03-18T20:00:02.571068Z","shell.execute_reply.started":"2025-03-18T20:00:02.567155Z","shell.execute_reply":"2025-03-18T20:00:02.570139Z"}},"outputs":[],"execution_count":18},{"cell_type":"code","source":"# def custom_test_generator(df, batch_size, target_size):\n# num_samples = len(df)\n# while True:\n# for offset in range(0, num_samples, batch_size):\n# batch_df = df.iloc[offset:offset+batch_size]\n# images = []\n# for _, row in batch_df.iterrows():\n# img = cv2.imread(row['filepath'])\n# if img is None:\n# continue\n# img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n# img = cv2.resize(img, target_size)\n# img = img.astype(\"float32\") / 255.0\n# images.append(img)\n# X = np.array(images)\n# yield X\n\n# test_gen = custom_test_generator(test_df, batch_size, (img_width, img_height))\n# predictions = model.predict(test_gen, steps=math.ceil(len(test_df) / batch_size))\n\n# submission_df = pd.DataFrame({\n# 'id': test_df['id'].astype(int),\n# 'label': predictions.flatten()\n# }).sort_values('id')\n# submission_df.to_csv('submission.csv', index=False)\n# print(\"Submission file saved as submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.371353Z","iopub.status.idle":"2025-03-18T19:28:45.371722Z","shell.execute_reply":"2025-03-18T19:28:45.371533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D, Dropout, Dense\n\n# model = Sequential([\n# Conv2D(32, (3,3), activation='relu', input_shape=(img_width, img_height, 3)),\n# BatchNormalization(),\n# MaxPooling2D(2,2),\n \n# Conv2D(64, (3,3), activation='relu'),\n# BatchNormalization(),\n# MaxPooling2D(2,2),\n \n# Conv2D(128, (3,3), activation='relu'),\n# BatchNormalization(),\n# MaxPooling2D(2,2),\n \n# Conv2D(128, (3,3), activation='relu'),\n# BatchNormalization(),\n# MaxPooling2D(2,2),\n \n# GlobalAveragePooling2D(),\n# Dropout(0.5),\n# Dense(512, activation='relu'),\n# Dense(1, activation='sigmoid')\n# ])\n\n\n\n# from tensorflow.keras.metrics import BinaryCrossentropy\n# model.compile(loss='binary_crossentropy',\n# optimizer=Adam(learning_rate=1e-4),\n# metrics=[BinaryCrossentropy(), 'accuracy'])\n\n\n\n# model.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.372991Z","iopub.status.idle":"2025-03-18T19:28:45.373289Z","shell.execute_reply":"2025-03-18T19:28:45.373164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras.applications import ResNet50\n# from tensorflow.keras import layers, models\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# img_width, img_height = 150, 150\n\n# base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))\n# base_model.trainable = False # Замораживаем базовую модель\n\n# model = models.Sequential([\n# base_model,\n# layers.GlobalAveragePooling2D(),\n# layers.Dense(512, activation='relu'),\n# layers.Dropout(0.5),\n# layers.Dense(1, activation='sigmoid') # Бинарная классификация: собака/кошка\n# ])\n\n# model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),\n# loss='binary_crossentropy',\n# metrics=['accuracy'])\n\n# model.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.374493Z","iopub.status.idle":"2025-03-18T19:28:45.37488Z","shell.execute_reply":"2025-03-18T19:28:45.374724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(\"Train samples:\", train_generator.samples)\n# print(\"Validation samples:\", validation_generator.samples)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.375962Z","iopub.status.idle":"2025-03-18T19:28:45.376235Z","shell.execute_reply":"2025-03-18T19:28:45.376119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import numpy as np\n# from tensorflow.keras.callbacks import EarlyStopping\n\n# steps_per_epoch = int(np.ceil(train_generator.samples / train_generator.batch_size))\n# validation_steps = int(np.ceil(validation_generator.samples / validation_generator.batch_size))\n\n# epochs = 20\n# early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\n\n# history = model.fit(\n# train_generator,\n# # steps_per_epoch=steps_per_epoch,\n# epochs=epochs,\n# validation_data=validation_generator,\n# # validation_steps=validation_steps,\n# callbacks=[early_stopping]\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.377162Z","iopub.status.idle":"2025-03-18T19:28:45.377529Z","shell.execute_reply":"2025-03-18T19:28:45.377368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import torch\n# import cv2\n# import numpy as np\n\n# model_yolo = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)\n# model_yolo.eval() \n\n# def yolo_crop(image):\n# results = model_yolo(image)\n# detections = results.xyxy[0].cpu().numpy() # каждая строка: [x1, y1, x2, y2, conf, cls]\n \n# crop_img = image \n# best_conf = 0\n# for det in detections:\n# conf = det[4]\n# cls = int(det[5])\n# label = model_yolo.names[cls]\n# if label in ['cat', 'dog'] and conf > best_conf:\n# best_conf = conf\n# x1, y1, x2, y2 = map(int, det[:4])\n# h, w, _ = image.shape\n# x1 = max(0, x1)\n# y1 = max(0, y1)\n# x2 = min(w, x2)\n# y2 = min(h, y2)\n# crop_img = image[y1:y2, x1:x2, :]\n# return crop_img\n\n# img_width, img_height = 224, 224\n\n# def custom_preprocessing_function(image):\n# if image.max() <= 1.0:\n# image_uint8 = (image * 255).astype(np.uint8)\n# else:\n# image_uint8 = image.astype(np.uint8)\n# cropped = yolo_crop(image_uint8)\n# resized = cv2.resize(cropped, (img_width, img_height))\n# normalized = resized / 255.0\n# return normalized\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.378576Z","iopub.status.idle":"2025-03-18T19:28:45.378859Z","shell.execute_reply":"2025-03-18T19:28:45.378752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import pandas as pd\n# from sklearn.model_selection import train_test_split\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# df = pd.DataFrame({'filename': os.listdir(train_dir)})\n# df = df[df['filename'].str.startswith(('cat', 'dog'))]\n# df['label'] = df['filename'].apply(lambda x: 'cat' if x.startswith('cat') else 'dog')\n# df['filepath'] = df['filename'].apply(lambda x: os.path.join(train_dir, x))\n\n# cats_df = df[df['label'] == 'cat'].sample(3000, random_state=42)\n# dogs_df = df[df['label'] == 'dog'].sample(3000, random_state=42)\n# df = pd.concat([cats_df, dogs_df]).reset_index(drop=True)\n\n# train_df, val_df = train_test_split(df, test_size=0.2, stratify=df['label'], random_state=42)\n\n# batch_size = 32\n\n# train_datagen = ImageDataGenerator(\n# preprocessing_function=custom_preprocessing_function,\n# rotation_range=40,\n# zoom_range=0.2,\n# horizontal_flip=True,\n# shear_range=0.2,\n# width_shift_range=0.2,\n# height_shift_range=0.2,\n# fill_mode='nearest'\n# )\n\n# val_datagen = ImageDataGenerator(\n# preprocessing_function=custom_preprocessing_function\n# )\n\n# train_generator = train_datagen.flow_from_dataframe(\n# dataframe=train_df,\n# x_col='filepath',\n# y_col='label',\n# target_size=(224, 224),\n# class_mode='binary',\n# batch_size=batch_size,\n# shuffle=True\n# )\n\n# validation_generator = val_datagen.flow_from_dataframe(\n# dataframe=val_df,\n# x_col='filepath',\n# y_col='label',\n# target_size=(224, 224),\n# class_mode='binary',\n# batch_size=batch_size,\n# shuffle=False\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.379735Z","iopub.status.idle":"2025-03-18T19:28:45.379984Z","shell.execute_reply":"2025-03-18T19:28:45.379881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import numpy as np\n# from tensorflow.keras.callbacks import EarlyStopping\n\n# steps_per_epoch = int(np.ceil(train_generator.samples / train_generator.batch_size))\n# validation_steps = int(np.ceil(validation_generator.samples / validation_generator.batch_size))\n\n# epochs = 20\n# early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\n\n# history = model.fit(\n# train_generator,\n# epochs=epochs,\n# validation_data=validation_generator,\n# callbacks=[early_stopping]\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.38089Z","iopub.status.idle":"2025-03-18T19:28:45.381259Z","shell.execute_reply":"2025-03-18T19:28:45.381105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# # test_dir = 'path_to_test_dir'\n# test_files = os.listdir(test_dir)\n# test_files.sort(key=lambda x: int(os.path.splitext(x)[0]))\n\n# test_df = pd.DataFrame({\n# 'filepath': [os.path.join(test_dir, file) for file in test_files],\n# 'id': [os.path.splitext(file)[0] for file in test_files]\n# })\n\n# test_datagen = ImageDataGenerator(\n# preprocessing_function=custom_preprocessing_function\n# )\n\n# test_generator = test_datagen.flow_from_dataframe(\n# dataframe=test_df,\n# x_col='filepath',\n# y_col=None, # метки отсутствуют\n# target_size=(img_width, img_height),\n# batch_size=batch_size,\n# shuffle=False,\n# class_mode=None\n# )\n\n# predictions = model.predict(test_generator, steps=len(test_generator))\n\n# submission_df = pd.DataFrame({\n# 'id': test_df['id'].astype(int),\n# 'label': predictions.flatten()\n# }).sort_values('id')\n\n# submission_df.to_csv('submission.csv', index=False)\n# print(\"Submission file saved as submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.382505Z","iopub.status.idle":"2025-03-18T19:28:45.382841Z","shell.execute_reply":"2025-03-18T19:28:45.382712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_files = os.listdir(test_dir)\n# test_files.sort(key=lambda x: int(os.path.splitext(x)[0]))\n\n# test_images = []\n# test_ids = []\n\n# # for file in test_files:\n# # img_path = os.path.join(test_dir, file)\n# # img = Image.open(img_path).resize((img_width, img_height))\n# # img = np.array(img) / 255.0\n# # if img.shape != (img_width, img_height, 3):\n# # img = np.stack((img,)*3, axis=-1)\n# # test_images.append(img)\n# # test_ids.append(os.path.splitext(file)[0])\n\n# test_images = np.array(test_images)\n\n# predictions = model.predict(test_images)\n\n# submission_df = pd.DataFrame({\n# 'id': test_ids,\n# 'label': predictions.flatten()\n# })\n# submission_df['id'] = submission_df['id'].astype(int)\n# submission_df = submission_df.sort_values('id')\n\n# # Сохраняем в CSV\n# submission_df.to_csv('submission.csv', index=False)\n# print(\"Submission file saved as submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.383525Z","iopub.status.idle":"2025-03-18T19:28:45.383872Z","shell.execute_reply":"2025-03-18T19:28:45.38371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# test_files = os.listdir(test_dir)\n# test_files.sort(key=lambda x: int(os.path.splitext(x)[0])) \n\n# test_df = pd.DataFrame({\n# 'filepath': [os.path.join(test_dir, file) for file in test_files],\n# 'id': [os.path.splitext(file)[0] for file in test_files]\n# })\n\n# test_datagen = ImageDataGenerator(rescale=1./255)\n\n# test_generator = test_datagen.flow_from_dataframe(\n# dataframe=test_df,\n# x_col='filepath',\n# y_col=None, \n# target_size=(img_width, img_height),\n# batch_size=batch_size, \n# shuffle=False, \n# class_mode=None\n# )\n\n# predictions = model.predict(test_generator, steps=len(test_generator))\n\n# submission_df = pd.DataFrame({\n# 'id': test_df['id'].astype(int),\n# 'label': predictions.flatten()\n# }).sort_values('id')\n\n# submission_df.to_csv('submission.csv', index=False)\n# print(\"Submission file saved as submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:28:45.385042Z","iopub.status.idle":"2025-03-18T19:28:45.385385Z","shell.execute_reply":"2025-03-18T19:28:45.385267Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}