{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 必要なライブラリのインポート\nimport zipfile\nimport os\nimport numpy as np\nimport cv2\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.applications import VGG16\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport random\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:12.392661Z","iopub.execute_input":"2024-07-23T06:18:12.393263Z","iopub.status.idle":"2024-07-23T06:18:12.407576Z","shell.execute_reply.started":"2024-07-23T06:18:12.393217Z","shell.execute_reply":"2024-07-23T06:18:12.406295Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv\n/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip\n/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip\n","output_type":"stream"}]},{"cell_type":"markdown","source":"データセットの解凍と前処理¶\n以下の処理を行う\n\nデータセットの解凍\nエッジ検出の関数定義\nデータジェネレータの定義と作成","metadata":{}},{"cell_type":"code","source":"# データセットのパス\ntrain_zip_path = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\ntest_zip_path = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\n\n# データセットの解凍\nwith zipfile.ZipFile(train_zip_path, 'r') as zip_ref:\n    zip_ref.extractall('/kaggle/working/train')\n\nwith zipfile.ZipFile(test_zip_path, 'r') as zip_ref:\n    zip_ref.extractall('/kaggle/working/test')\n    \n# 画像が存在するディレクトリ\ntrain_dir = '/kaggle/working/train/train'\ntest_dir = '/kaggle/working/test/test'","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:12.410226Z","iopub.execute_input":"2024-07-23T06:18:12.410689Z","iopub.status.idle":"2024-07-23T06:18:26.043634Z","shell.execute_reply.started":"2024-07-23T06:18:12.41065Z","shell.execute_reply":"2024-07-23T06:18:26.042361Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"markdown","source":"エッジ検出画像の作成 今回はオリジナルの訓練用画像をもとにエッジ検出画像を作成し、一緒に学習させる。エッジ検出にはopenCVのCannyエッジ検出を利用した。","metadata":{}},{"cell_type":"code","source":"# エッジ検出用の関数\ndef preprocess_image(img_path):\n    img = cv2.imread(img_path)\n    if img is None:\n        return None, None\n    \n    img = cv2.resize(img, (128, 128))  # リサイズ\n    \n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    edges = cv2.Canny(gray, 100, 200)\n    edges = cv2.cvtColor(edges, cv2.COLOR_GRAY2BGR)  # 3チャンネルに変換\n\n    return img, edges","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.045224Z","iopub.execute_input":"2024-07-23T06:18:26.045656Z","iopub.status.idle":"2024-07-23T06:18:26.053395Z","shell.execute_reply.started":"2024-07-23T06:18:26.045614Z","shell.execute_reply":"2024-07-23T06:18:26.052239Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"markdown","source":"データジェネレータの定義¶ データジェネレータはメモリ節約のために設定。バッチサイズごとにオリジナルの訓練用画像とそれを元とするエッジ検出画像を生成する。","metadata":{}},{"cell_type":"code","source":"# データジェネレータの定義\ndef image_edge_generator(file_list, batch_size=32):\n    while True:\n        batch_images = []\n        batch_edges = []\n        batch_labels = []\n        \n        for _ in range(batch_size):\n            img_file = random.choice(file_list)\n            img_path = os.path.join(train_dir, img_file)\n            \n            img, edges = preprocess_image(img_path)\n            if img is not None and edges is not None:\n                label = 1 if 'dog' in img_file else 0\n                batch_images.append(img)\n                batch_edges.append(edges)\n                batch_labels.append(label)\n        \n        if len(batch_images) > 0:\n            batch_images = np.array(batch_images)\n            batch_edges = np.array(batch_edges)\n            batch_labels = np.array(batch_labels)\n            \n            augmented_images = np.concatenate((batch_images, batch_edges), axis=0)\n            augmented_labels = np.concatenate((batch_labels, batch_labels), axis=0)\n            \n            yield augmented_images / 255.0, augmented_labels","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.056794Z","iopub.execute_input":"2024-07-23T06:18:26.057535Z","iopub.status.idle":"2024-07-23T06:18:26.070946Z","shell.execute_reply.started":"2024-07-23T06:18:26.057495Z","shell.execute_reply":"2024-07-23T06:18:26.069453Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"# train_dir内のすべてのファイル名を取得\nfile_list = [f for f in os.listdir(train_dir) if os.path.isfile(os.path.join(train_dir, f))]\n\n# データジェネレータの作成\nbatch_size = 32\ntrain_gen = image_edge_generator(file_list, batch_size)\nvalidation_gen = image_edge_generator(file_list, batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.072726Z","iopub.execute_input":"2024-07-23T06:18:26.07311Z","iopub.status.idle":"2024-07-23T06:18:26.303391Z","shell.execute_reply.started":"2024-07-23T06:18:26.073076Z","shell.execute_reply":"2024-07-23T06:18:26.302156Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"markdown","source":"モデルの構築と学習 予め学習済みのVGG16を使用する。","metadata":{}},{"cell_type":"code","source":"# VGG16モデルのロード(トップ層は含まない)\nbase_model = VGG16(weights='imagenet', include_top=False, input_shape=(128, 128, 3))\n​\n# 新しい分類層の追加\nx = base_model.output\nx = Flatten()(x)\nx = Dense(512, activation='relu')(x)\nx = Dropout(0.5)(x)\npredictions = Dense(1, activation='sigmoid')(x)\n​\n# 新しいモデルの定義\nmodel = Model(inputs=base_model.input, outputs=predictions)\n​\n# すでに学習済みの層は凍結\nfor layer in base_model.layers:\n    layer.trainable = False\n​\n# モデルのコンパイル\nmodel.compile(optimizer=Adam(learning_rate=0.001),\n              loss='binary_crossentropy',\n              metrics=['accuracy', AUC()])\n​\nmodel.summary() ","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.304668Z","iopub.execute_input":"2024-07-23T06:18:26.30512Z","iopub.status.idle":"2024-07-23T06:18:26.3154Z","shell.execute_reply.started":"2024-07-23T06:18:26.305083Z","shell.execute_reply":"2024-07-23T06:18:26.313454Z"},"trusted":true},"execution_count":22,"outputs":[{"traceback":["\u001b[0;36m  Cell \u001b[0;32mIn[22], line 3\u001b[0;36m\u001b[0m\n\u001b[0;31m    ​\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid non-printable character U+200B\n"],"ename":"SyntaxError","evalue":"invalid non-printable character U+200B (3819755050.py, line 3)","output_type":"error"}]},{"cell_type":"markdown","source":"モデルの学習 エポックごとに学習と検証を行う","metadata":{}},{"cell_type":"code","source":"# 早期終了コールバック\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n\n# モデルの学習\nhistory = model.fit(train_gen, \n                    steps_per_epoch=len(file_list) // batch_size, \n                    epochs=50, \n                    validation_data=validation_gen, \n                    validation_steps=len(file_list) // batch_size,\n                    callbacks=[early_stopping])","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.316588Z","iopub.status.idle":"2024-07-23T06:18:26.317057Z","shell.execute_reply.started":"2024-07-23T06:18:26.316826Z","shell.execute_reply":"2024-07-23T06:18:26.316844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習過程のプロット\ndef plot_training_history(history):\n    # 損失のプロット\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.title('Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    # 精度のプロット\n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['accuracy'], label='Train Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n    plt.title('Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n\n    plt.tight_layout()\n    plt.show()\n\n# プロットの実行\nplot_training_history(history)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.318639Z","iopub.status.idle":"2024-07-23T06:18:26.319229Z","shell.execute_reply.started":"2024-07-23T06:18:26.318915Z","shell.execute_reply":"2024-07-23T06:18:26.318941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"モデルの評価\n訓練したモデルにテストデータを入れて犬か猫かを予測する","metadata":{}},{"cell_type":"code","source":"# テストデータの前処理\ntest_images = []\n\nfor filename in os.listdir(test_dir):\n    img_path = os.path.join(test_dir, filename)\n    \n    if not os.path.isfile(img_path):\n        continue\n    \n    img = cv2.imread(img_path)\n    \n    if img is None:\n        print(f\"Warning: Could not read image {img_path}\")\n        continue\n    \n    img = cv2.resize(img, (128, 128))  # リサイズ\n    test_images.append(img)\n\ntest_images = np.array(test_images)\ntest_images = test_images / 255.0  # 標準化\n\n# 予測\npredictions = model.predict(test_images)\nnp.set_printoptions(precision=7, suppress=True)\npredictions","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.320995Z","iopub.status.idle":"2024-07-23T06:18:26.321586Z","shell.execute_reply.started":"2024-07-23T06:18:26.321249Z","shell.execute_reply":"2024-07-23T06:18:26.321272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# 予測結果のDataFrameを作成\npred_df = pd.DataFrame({'id': range(1, len(predictions) + 1), 'label': predictions.flatten()})\npd.set_option('display.float_format', '{:.7f}'.format)\n\n# データフレームの表示\nprint(pred_pd.head())","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.323131Z","iopub.status.idle":"2024-07-23T06:18:26.323559Z","shell.execute_reply.started":"2024-07-23T06:18:26.323362Z","shell.execute_reply":"2024-07-23T06:18:26.323379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"提出用ファイルに結果を保存\n出力された結果を、サブミット用のcsvファイルに保存","metadata":{}},{"cell_type":"code","source":"# 提出用データの準備\nsub = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')  # ひな形ファイルを読み込む\nprint(sub.shape)\n\n# 提出用データのラベル列を予測値で上書き\nsub['label'] = pred_pd['label']\n\n# 結果を表示\nprint(sub.head())\n\n# CSVファイルに保存\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-23T06:18:26.324998Z","iopub.status.idle":"2024-07-23T06:18:26.325568Z","shell.execute_reply.started":"2024-07-23T06:18:26.325294Z","shell.execute_reply":"2024-07-23T06:18:26.325316Z"},"trusted":true},"execution_count":null,"outputs":[]}]}