{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 必要なライブラリのインポート\nimport zipfile\nimport os\nimport numpy as np\nimport cv2\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.applications import VGG16\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport random\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-25T12:41:39.128861Z","iopub.execute_input":"2024-07-25T12:41:39.12949Z","iopub.status.idle":"2024-07-25T12:41:52.818036Z","shell.execute_reply.started":"2024-07-25T12:41:39.129459Z","shell.execute_reply":"2024-07-25T12:41:52.81698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**以下のコードでは、KaggleのDogs vs Cats ReduxデータセットのZIPファイルを解凍し、トレーニングデータとテストデータが存在するディレクトリを指定しています。**\n データセットの解凍\n エッジ検出の関数定義\n データジェネレータの定義と作成","metadata":{}},{"cell_type":"code","source":"# データセットのパス\ntrain_zip_path = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\ntest_zip_path = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\n\n# データセットの解凍\nwith zipfile.ZipFile(train_zip_path, 'r') as zip_ref:\n    zip_ref.extractall('/kaggle/working/train')\n\nwith zipfile.ZipFile(test_zip_path, 'r') as zip_ref:\n    zip_ref.extractall('/kaggle/working/test')\n    \n# 画像が存在するディレクトリ\ntrain_dir = '/kaggle/working/train/train'\ntest_dir = '/kaggle/working/test/test'","metadata":{"execution":{"iopub.status.busy":"2024-07-25T12:41:52.819809Z","iopub.execute_input":"2024-07-25T12:41:52.820331Z","iopub.status.idle":"2024-07-25T12:42:08.477276Z","shell.execute_reply.started":"2024-07-25T12:41:52.820305Z","shell.execute_reply":"2024-07-25T12:42:08.476338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**エッジ検出用の関数preprocess_imageは、以下のステップを実行しています**\n\n画像を読み込む\n画像をリサイズする\n画像をグレースケールに変換する\nエッジ検出を実行する\nエッジ検出の結果を3チャンネルに変換する","metadata":{}},{"cell_type":"code","source":"# エッジ検出用の関数\ndef preprocess_image(img_path):\n    img = cv2.imread(img_path)\n    if img is None:\n        return None, None\n    \n    img = cv2.resize(img, (128, 128))  # リサイズ\n    \n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    edges = cv2.Canny(gray, 100, 200)\n    edges = cv2.cvtColor(edges, cv2.COLOR_GRAY2BGR)  # 3チャンネルに変換\n\n    return img, edges","metadata":{"execution":{"iopub.status.busy":"2024-07-25T12:42:08.478356Z","iopub.execute_input":"2024-07-25T12:42:08.478616Z","iopub.status.idle":"2024-07-25T12:42:08.484821Z","shell.execute_reply.started":"2024-07-25T12:42:08.478593Z","shell.execute_reply":"2024-07-25T12:42:08.483654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**image_edge_generator関数は、以下の手順でバッチ処理を行うジェネレータを定義しています**\n\n指定されたファイルリストからランダムに画像を選びます。\n画像を読み込んでエッジ検出を行います。\n画像が読み込まれ、エッジ検出が成功した場合、その画像のラベル（犬なら1、猫なら0）を設定します。\n指定されたバッチサイズに達するまでこれを繰り返します。\n画像とエッジ検出結果を含むバッチを生成します。\n生成されたバッチを正規化し、ラベルと共にジェネレータとして返します。","metadata":{}},{"cell_type":"code","source":"# データジェネレータの定義\ndef image_edge_generator(file_list, batch_size=32):\n    while True:\n        batch_images = []\n        batch_edges = []\n        batch_labels = []\n        \n        for _ in range(batch_size):\n            img_file = random.choice(file_list)\n            img_path = os.path.join(train_dir, img_file)\n            \n            img, edges = preprocess_image(img_path)\n            if img is not None and edges is not None:\n                label = 1 if 'dog' in img_file else 0\n                batch_images.append(img)\n                batch_edges.append(edges)\n                batch_labels.append(label)\n        \n        if len(batch_images) > 0:\n            batch_images = np.array(batch_images)\n            batch_edges = np.array(batch_edges)\n            batch_labels = np.array(batch_labels)\n            \n            augmented_images = np.concatenate((batch_images, batch_edges), axis=0)\n            augmented_labels = np.concatenate((batch_labels, batch_labels), axis=0)\n            \n            yield augmented_images / 255.0, augmented_labels","metadata":{"execution":{"iopub.status.busy":"2024-07-25T12:42:08.486989Z","iopub.execute_input":"2024-07-25T12:42:08.487291Z","iopub.status.idle":"2024-07-25T12:42:08.496734Z","shell.execute_reply.started":"2024-07-25T12:42:08.487268Z","shell.execute_reply":"2024-07-25T12:42:08.495792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dir内のすべてのファイル名を取得\nfile_list = [f for f in os.listdir(train_dir) if os.path.isfile(os.path.join(train_dir, f))]\n\n# データジェネレータの作成\nbatch_size = 32\ntrain_gen = image_edge_generator(file_list, batch_size)\nvalidation_gen = image_edge_generator(file_list, batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-07-25T12:42:08.497956Z","iopub.execute_input":"2024-07-25T12:42:08.498263Z","iopub.status.idle":"2024-07-25T12:42:08.71117Z","shell.execute_reply.started":"2024-07-25T12:42:08.498216Z","shell.execute_reply":"2024-07-25T12:42:08.71039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"下記のコードでは、VGG16モデルをベースとして使用し、トップ層を含まない形でモデルをロードし、新しい分類層を追加しています。その後、VGG16の既存の層を凍結して新しい層のみを学習するように設定し、モデルをコンパイルしています。","metadata":{}},{"cell_type":"code","source":"# VGG16モデルのロード(トップ層は含まない)\nbase_model = VGG16(weights='imagenet', include_top=False, input_shape=(128, 128, 3))\n\n# 新しい分類層の追加\nx = base_model.output\nx = Flatten()(x)\nx = Dense(512, activation='relu')(x)\nx = Dropout(0.5)(x)\npredictions = Dense(1, activation='sigmoid')(x)\n\n# 新しいモデルの定義\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\n# すでに学習済みの層は凍結\nfor layer in base_model.layers:\n    layer.trainable = False\n\n# モデルのコンパイル\nmodel.compile(optimizer=Adam(learning_rate=0.001),\n              loss='binary_crossentropy',\n              metrics=['accuracy', AUC()])\n\nmodel.summary() ","metadata":{"execution":{"iopub.status.busy":"2024-07-25T12:42:08.712364Z","iopub.execute_input":"2024-07-25T12:42:08.712654Z","iopub.status.idle":"2024-07-25T12:42:10.696986Z","shell.execute_reply.started":"2024-07-25T12:42:08.71263Z","shell.execute_reply":"2024-07-25T12:42:10.696076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**早期終了コールバック（EarlyStopping）を使って、モデルの学習を行うコードは以下のように設定されています。**これにより、検証損失（val_loss）が改善しない場合に訓練を早期に停止し、最良のモデルの重みを復元します。","metadata":{}},{"cell_type":"code","source":"# 早期終了コールバック\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n\n# モデルの学習\nhistory = model.fit(train_gen, \n                    steps_per_epoch=len(file_list) // batch_size, \n                    epochs=50, \n                    validation_data=validation_gen, \n                    validation_steps=len(file_list) // batch_size,\n                    callbacks=[early_stopping])","metadata":{"execution":{"iopub.status.busy":"2024-07-25T12:42:10.698529Z","iopub.execute_input":"2024-07-25T12:42:10.698881Z","iopub.status.idle":"2024-07-25T14:28:23.19268Z","shell.execute_reply.started":"2024-07-25T12:42:10.698846Z","shell.execute_reply":"2024-07-25T14:28:23.191772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習過程のプロット\ndef plot_training_history(history):\n    # 損失のプロット\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Validation Loss')\n    plt.title('Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    # 精度のプロット\n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['accuracy'], label='Train Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n    plt.title('Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n\n    plt.tight_layout()\n    plt.show()\n\n# プロットの実行\nplot_training_history(history)","metadata":{"execution":{"iopub.status.busy":"2024-07-25T14:28:23.193983Z","iopub.execute_input":"2024-07-25T14:28:23.194324Z","iopub.status.idle":"2024-07-25T14:28:23.920737Z","shell.execute_reply.started":"2024-07-25T14:28:23.19429Z","shell.execute_reply":"2024-07-25T14:28:23.919823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**テストデータに対して予測を行うためのコードは、以下のように機能しています。コードの流れは次の通りです**\n\nテスト画像の読み込みと前処理:\n\nテスト画像のパスを取得し、各画像を読み込みます。\n画像をリサイズし、リストに追加します。\n最後に、リストをNumPy配列に変換し、ピクセル値を0-1の範囲に標準化します。\n予測の実行:\n\nモデルを使用して、前処理済みのテスト画像に対して予測を行います。\n予測結果の表示:\n\n予測結果の精度を制御し、結果を表示します。","metadata":{}},{"cell_type":"code","source":"# テストデータの前処理\ntest_images = []\n\nfor filename in os.listdir(test_dir):\n    img_path = os.path.join(test_dir, filename)\n    \n    if not os.path.isfile(img_path):\n        continue\n    \n    img = cv2.imread(img_path)\n    \n    if img is None:\n        print(f\"Warning: Could not read image {img_path}\")\n        continue\n    \n    img = cv2.resize(img, (128, 128))  # リサイズ\n    test_images.append(img)\n\ntest_images = np.array(test_images)\ntest_images = test_images / 255.0  # 標準化\n\n# 予測\npredictions = model.predict(test_images)\nnp.set_printoptions(precision=7, suppress=True)\npredictions","metadata":{"execution":{"iopub.status.busy":"2024-07-25T14:28:23.921955Z","iopub.execute_input":"2024-07-25T14:28:23.922296Z","iopub.status.idle":"2024-07-25T14:29:20.497721Z","shell.execute_reply.started":"2024-07-25T14:28:23.922266Z","shell.execute_reply":"2024-07-25T14:29:20.496517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# 予測結果のDataFrameを作成\npred_df = pd.DataFrame({'id': range(1, len(predictions) + 1), 'label': predictions.flatten()})\npd.set_option('display.float_format', '{:.7f}'.format)\n\n# データフレームの表示\nprint(pred_pd.head())","metadata":{"execution":{"iopub.status.busy":"2024-07-25T14:29:20.50052Z","iopub.execute_input":"2024-07-25T14:29:20.500844Z","iopub.status.idle":"2024-07-25T14:29:21.220798Z","shell.execute_reply.started":"2024-07-25T14:29:20.500817Z","shell.execute_reply":"2024-07-25T14:29:21.219443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 提出用ファイルに結果を保存\n出力された結果を、サブミット用のcsvファイルに保存","metadata":{}},{"cell_type":"code","source":"# 提出用データの準備\nsub = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')  # ひな形ファイルを読み込む\nprint(sub.shape)\n\n# 提出用データのラベル列を予測値で上書き\nsub['label'] = pred_pd['label']\n\n# 結果を表示\nprint(sub.head())\n\n# CSVファイルに保存\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-25T14:29:21.221611Z","iopub.status.idle":"2024-07-25T14:29:21.222011Z","shell.execute_reply.started":"2024-07-25T14:29:21.221836Z","shell.execute_reply":"2024-07-25T14:29:21.221851Z"},"trusted":true},"execution_count":null,"outputs":[]}]}