{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:02.717211Z","iopub.execute_input":"2025-07-11T08:14:02.717527Z","iopub.status.idle":"2025-07-11T08:14:02.724333Z","shell.execute_reply.started":"2025-07-11T08:14:02.717504Z","shell.execute_reply":"2025-07-11T08:14:02.723717Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"!pip install torch torchvision Pillow\n!pip install fastai","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:02.726292Z","iopub.execute_input":"2025-07-11T08:14:02.726718Z","iopub.status.idle":"2025-07-11T08:14:09.091298Z","shell.execute_reply.started":"2025-07-11T08:14:02.726700Z","shell.execute_reply":"2025-07-11T08:14:09.090582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\nimport shutil\nimport zipfile\nimport tempfile\n\nfrom fastai.vision.all import *\nimport albumentations as Alb\nimport PIL\nimport torch \nfrom pathlib import Path\nfrom torchvision.models import resnext50_32x4d, ResNeXt50_32X4D_Weights\nfrom fastai.vision.all import vision_learner, error_rate, accuracy\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:09.092714Z","iopub.execute_input":"2025-07-11T08:14:09.092943Z","iopub.status.idle":"2025-07-11T08:14:09.099849Z","shell.execute_reply.started":"2025-07-11T08:14:09.092920Z","shell.execute_reply":"2025-07-11T08:14:09.099168Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## zipファイルの展開","metadata":{}},{"cell_type":"code","source":"#訓練データの解凍\nzip_path_train = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\ntemp_dir_train = tempfile.TemporaryDirectory()\nwith zipfile.ZipFile(zip_path_train, 'r') as zip_ref:\n    zip_ref.extractall(temp_dir_train.name)\ntrain_path = Path(temp_dir_train.name) / 'train'\nval_path = Path(temp_dir_train.name) / 'val'  # 今は使わなくてもOK\n\n# テストデータの解凍\nzip_path_test = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\ntemp_dir_test = tempfile.TemporaryDirectory()\nwith zipfile.ZipFile(zip_path_test, 'r') as zip_ref:\n    zip_ref.extractall(temp_dir_test.name)\ntest_path = Path(temp_dir_test.name) / 'test'\n\n# train フォルダ内ファイル一覧\nfile_list_train = list(train_path.iterdir())\n\n# 猫と犬のファイル名パターンでカウント\ncat_img_train = [f for f in file_list_train if f.name.startswith(\"cat\")]\ndog_img_train = [f for f in file_list_train if f.name.startswith(\"dog\")]\n\nprint(f\"Cat_img_train: {len(cat_img_train)}\")\nprint(f\"Dog_img_train: {len(dog_img_train)}\")\nprint(f\"Total_img_train: {len(file_list_train)}\")\n\n# val ディレクトリの中身確認\ncat_img_val = val_path / 'cat'\ndog_img_val = val_path / 'dog'\n\nif not cat_img_val.exists() or len(list(cat_img_val.iterdir())) == 0:\n    print(\"val/cat フォルダには画像が入っていません\")\nelse:\n    print(f\"val/cat フォルダに {len(list(cat_img_val.iterdir()))} 枚の画像があります\")\n\nif not dog_img_val.exists() or len(list(dog_img_val.iterdir())) == 0:\n    print(\"val/dog フォルダには画像が入っていません\")\nelse:\n    print(f\"val/dog フォルダに {len(list(dog_img_val.iterdir()))} 枚の画像があります\")\n\n# test データ確認\ntest_files = list(test_path.iterdir())\nprint(f\"Total_img_test: {len(test_files)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:09.100562Z","iopub.execute_input":"2025-07-11T08:14:09.100780Z","iopub.status.idle":"2025-07-11T08:14:19.058633Z","shell.execute_reply.started":"2025-07-11T08:14:09.100755Z","shell.execute_reply":"2025-07-11T08:14:19.057820Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 不良画像の削除","metadata":{}},{"cell_type":"code","source":"\n# 不良画像リスト\nbad_images = [\n    'dog.10797.jpg', 'dog.10747.jpg', 'dog.10237.jpg', 'dog.9517.jpg',\n    'dog.8736.jpg', 'dog.5604.jpg', 'dog.1043.jpg', 'cat.4338.jpg',\n    'dog.10161.jpg', 'dog.10190.jpg'\n]\n\n# train_path は先ほどのコードで定義した Path オブジェクト\nremoved_count = 0\nfor img in bad_images:\n    img_path = train_path / img\n    if img_path.exists():\n        img_path.unlink()  # ファイル削除\n        removed_count += 1\n        print(f\"削除しました: {img}\")\n    else:\n        print(f\"見つかりませんでした: {img}\")\n\nprint(f\"\\n合計 {removed_count} 件の不良画像を削除しました\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:19.060193Z","iopub.execute_input":"2025-07-11T08:14:19.060534Z","iopub.status.idle":"2025-07-11T08:14:19.073234Z","shell.execute_reply.started":"2025-07-11T08:14:19.060513Z","shell.execute_reply":"2025-07-11T08:14:19.072535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 訓練データにラベルを付ける","metadata":{}},{"cell_type":"code","source":"#クラス別ディレクトリの作成\ncat_dir_train = os.path.join(train_path, 'cat')\ndog_dir_train = os.path.join(train_path, 'dog')\nos.makedirs(cat_dir_train, exist_ok=True)\nos.makedirs(dog_dir_train, exist_ok=True)\n\nfor filename in os.listdir(train_path):\n    filepath = os.path.join(train_path, filename)\n    if os.path.isfile(filepath):\n        if filename.startswith('cat'):\n            shutil.move(filepath, os.path.join(cat_dir_train, filename))\n        elif filename.startswith('dog'):\n            shutil.move(filepath, os.path.join(dog_dir_train, filename))\n\nprint(f'分類完了: cat={len(os.listdir(cat_dir_train))}, dog={len(os.listdir(dog_dir_train))}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:19.074189Z","iopub.execute_input":"2025-07-11T08:14:19.074536Z","iopub.status.idle":"2025-07-11T08:14:20.156835Z","shell.execute_reply.started":"2025-07-11T08:14:19.074507Z","shell.execute_reply":"2025-07-11T08:14:20.156243Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データセットの前処理","metadata":{}},{"cell_type":"code","source":"# データ拡張の定義\nclass AlbTransform(Transform):\n    def __init__(self, aug): self.aug = aug\n    def encodes(self, img: PILImage):\n        aug_img = self.aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)\n\n# Albumentationsのデータ拡張（ぼかし含む）\ndef get_augs():\n    return Alb.Compose([\n        Alb.ShiftScaleRotate(rotate_limit=20, border_mode=0),\n        Alb.Transpose(),\n        Alb.HorizontalFlip(),  # ← 修正\n        Alb.RandomRotate90(),\n        Alb.RandomBrightnessContrast(),\n        Alb.HueSaturationValue(hue_shift_limit=5, sat_shift_limit=5, val_shift_limit=5),\n        Alb.CoarseDropout(),\n        Alb.OneOf([\n            Alb.Blur(blur_limit=3, p=0.5),              # 単純なぼかし\n            Alb.GaussianBlur(blur_limit=(3, 5), p=0.5)  # ガウスぼかし\n        ], p=0.5)  # 50%の確率でどちらかを適用\n    ])\n\n# 画像1枚ごとの拡張処理\nitem_tfms = [Resize(224), AlbTransform(get_augs())]\n\n# バッチ単位の前処理\nbatch_tfms = [Normalize.from_stats(*imagenet_stats), *aug_transforms()]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:20.157593Z","iopub.execute_input":"2025-07-11T08:14:20.157824Z","iopub.status.idle":"2025-07-11T08:14:20.169853Z","shell.execute_reply.started":"2025-07-11T08:14:20.157797Z","shell.execute_reply":"2025-07-11T08:14:20.169105Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データローダーの作成と確認","metadata":{}},{"cell_type":"code","source":"# 画像パス\npath = Path(train_path)  # train フォルダは cat/ と dog/ を含む\n\n# データローダーの作成\ndls = ImageDataLoaders.from_folder(\n    path,\n    train='.', \n    valid_pct=0.2,\n    seed=42,\n    item_tfms=item_tfms,\n    batch_tfms=batch_tfms,\n    bs=64\n)\n\n\n# 🔹 Trainデータの表示\nprint(\"🔍 Train データ（ラベル付き）\")\ndls.train.show_batch(max_n=9, figsize=(8,8), title='Train')\n\n# 🔹 Validationデータの表示\nprint(\"🔍 Validation データ（ラベル付き）\")\ndls.valid.show_batch(max_n=9, figsize=(8,8), title='Validation')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:20.170561Z","iopub.execute_input":"2025-07-11T08:14:20.170813Z","iopub.status.idle":"2025-07-11T08:14:25.876569Z","shell.execute_reply.started":"2025-07-11T08:14:20.170789Z","shell.execute_reply":"2025-07-11T08:14:25.875761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 枚数の確認\nprint(f\"データ数確認:\")\nprint(f\"Train データ数: {len(dls.train_ds)} 枚\")\nprint(f\"Valid データ数: {len(dls.valid_ds)} 枚\")\n\n# Train データセットからランダムに5枚表示\nprint(\"Train データ\")\nfor img, lbl in random.sample(list(dls.train_ds), 5):\n    print(f\"ラベルID: {lbl}, クラス名: {dls.vocab[lbl]}\")\n    display(img.to_thumb(128, 128))\n\n# Valid データセットからランダムに5枚表示\nprint(\"\\nValidation データ\")\nfor img, lbl in random.sample(list(dls.valid_ds), 5):\n    print(f\"ラベルID: {lbl}, クラス名: {dls.vocab[lbl]}\")\n    display(img.to_thumb(128, 128))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:14:25.877477Z","iopub.execute_input":"2025-07-11T08:14:25.877947Z","iopub.status.idle":"2025-07-11T08:15:08.407653Z","shell.execute_reply.started":"2025-07-11T08:14:25.877926Z","shell.execute_reply":"2025-07-11T08:15:08.406979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## モデルの作成","metadata":{}},{"cell_type":"markdown","source":"## ResNet101","metadata":{}},{"cell_type":"code","source":"# モデルの作成\nlearn = vision_learner(dls, resnet101, metrics=[error_rate, accuracy])\n\n#最適な学習率の推定\nlearn_RN101 = learn.lr_find()\nprint(learn_RN101) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 学習実行\nlearn.fit_one_cycle(5, lr_max=learn_RN101.valley)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:15:50.404853Z","iopub.execute_input":"2025-07-11T08:15:50.405102Z","iopub.status.idle":"2025-07-11T08:28:26.035502Z","shell.execute_reply.started":"2025-07-11T08:15:50.405080Z","shell.execute_reply":"2025-07-11T08:28:26.034656Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テストデータの準備","metadata":{}},{"cell_type":"code","source":"# テストデータの準備\nprint('Testing', len(test_files), 'items')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:28:26.036582Z","iopub.execute_input":"2025-07-11T08:28:26.036832Z","iopub.status.idle":"2025-07-11T08:28:26.042042Z","shell.execute_reply.started":"2025-07-11T08:28:26.036808Z","shell.execute_reply":"2025-07-11T08:28:26.041435Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テストデータセットローダーの作成","metadata":{}},{"cell_type":"code","source":"# データ拡張なしの test_dl を明示的に作る\ntst_dl = dls.test_dl(test_files, with_labels=False, shuffle=False)\ntst_dl.show_batch(max_n=12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:28:26.042772Z","iopub.execute_input":"2025-07-11T08:28:26.043526Z","iopub.status.idle":"2025-07-11T08:28:27.361124Z","shell.execute_reply.started":"2025-07-11T08:28:26.043502Z","shell.execute_reply":"2025-07-11T08:28:27.360300Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 推論","metadata":{}},{"cell_type":"code","source":"# 推論実行\nstartTime = time.time()\nprobs, _ = learn.tta(dl=tst_dl, n=5, use_max=False)\nprint('TTA in:', time.time()-startTime, 'secs')\n\n# ラベル取得（最大確率のインデックス）\npreds = probs.argmax(dim=1)\nclass_names = learn.dls.vocab","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:28:27.362187Z","iopub.execute_input":"2025-07-11T08:28:27.362468Z","iopub.status.idle":"2025-07-11T08:33:33.939890Z","shell.execute_reply.started":"2025-07-11T08:28:27.362447Z","shell.execute_reply":"2025-07-11T08:33:33.938819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ランダムに12枚表示\nidxs = random.sample(range(len(preds)), 12)\nfig, axs = plt.subplots(3, 4, figsize=(12, 9))\n\nfor ax, idx in zip(axs.flatten(), idxs):\n    img = PILImage.create(tst_dl.items[idx])\n    pred_label = preds[idx].item()\n    pred_class = class_names[pred_label]\n    prob = probs[idx][pred_label].item()\n    ax.imshow(img)\n    ax.set_title(f'{pred_class} ({pred_label})\\nprob: {prob:.2f}')\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:33:33.941376Z","iopub.execute_input":"2025-07-11T08:33:33.941749Z","iopub.status.idle":"2025-07-11T08:33:35.107522Z","shell.execute_reply.started":"2025-07-11T08:33:33.941713Z","shell.execute_reply":"2025-07-11T08:33:35.106449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 提出用 DataFrame の作成\nsubm_df = pd.DataFrame()\nsubm_df['id'] = [item.stem for item in tst_dl.items]        # ファイル名（拡張子なし）\nsubm_df['label'] = preds.numpy()                            # argmax によるラベル（0=cat, 1=dog）\n\n# CSV ファイルとして保存\nsubm_df.to_csv('submission.csv', index=False)\nprint(\"✅ submission.csv を保存しました。\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:33:35.108484Z","iopub.execute_input":"2025-07-11T08:33:35.108687Z","iopub.status.idle":"2025-07-11T08:33:35.151178Z","shell.execute_reply.started":"2025-07-11T08:33:35.108671Z","shell.execute_reply":"2025-07-11T08:33:35.150641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ソフトマックスでクラス1（dog）の確率を取得\nprobs_softmax = torch.softmax(probs, dim=1)  # shape: [N, 2]\ndog_probs = probs_softmax[:, 1].clip(0.005, 0.995).numpy()  # 過剰な0,1を防ぐ\n\n# DataFrame の作成\nsubm_softmax_df = pd.DataFrame({\n    'id': [item.stem for item in tst_dl.items],\n    'label': dog_probs\n})\n\n# CSV保存\nsubm_softmax_df.to_csv('submission_softmax.csv', index=False)\nprint(\"✅ submission_softmax.csv を保存しました。\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T08:33:35.151986Z","iopub.execute_input":"2025-07-11T08:33:35.152258Z","iopub.status.idle":"2025-07-11T08:33:35.183505Z","shell.execute_reply.started":"2025-07-11T08:33:35.152235Z","shell.execute_reply":"2025-07-11T08:33:35.182927Z"}},"outputs":[],"execution_count":null}]}