{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:53:45.374587Z","iopub.execute_input":"2025-07-25T05:53:45.375177Z","iopub.status.idle":"2025-07-25T05:53:45.631927Z","shell.execute_reply.started":"2025-07-25T05:53:45.375153Z","shell.execute_reply":"2025-07-25T05:53:45.631205Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport time\nfrom fastai.vision.all import *\nimport zipfile\nimport albumentations as Alb\nimport random\nimport shutil\nimport PIL\nimport torch \nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom fastai.vision.augment import RandTransform\nfrom albumentations.pytorch import ToTensorV2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:53:45.633125Z","iopub.execute_input":"2025-07-25T05:53:45.633393Z","iopub.status.idle":"2025-07-25T05:53:57.933733Z","shell.execute_reply.started":"2025-07-25T05:53:45.633376Z","shell.execute_reply":"2025-07-25T05:53:57.932971Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## zipファイルの展開","metadata":{}},{"cell_type":"code","source":"#訓練データの解凍\nzip_path_train = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\ntemp_dir_train = tempfile.TemporaryDirectory()\nwith zipfile.ZipFile(zip_path_train, 'r') as zip_ref:\n    zip_ref.extractall(temp_dir_train.name)\ntrain_path = Path(temp_dir_train.name) / 'train'\nval_path = Path(temp_dir_train.name) / 'val'  # 今は使わなくてもOK\n\n# テストデータの解凍\nzip_path_test = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\ntemp_dir_test = tempfile.TemporaryDirectory()\nwith zipfile.ZipFile(zip_path_test, 'r') as zip_ref:\n    zip_ref.extractall(temp_dir_test.name)\ntest_path = Path(temp_dir_test.name) / 'test'\n\n# train フォルダ内ファイル一覧\nfile_list_train = list(train_path.iterdir())\n\n# 猫と犬のファイル名パターンでカウント\ncat_img_train = [f for f in file_list_train if f.name.startswith(\"cat\")]\ndog_img_train = [f for f in file_list_train if f.name.startswith(\"dog\")]\n\nprint(f\"Cat_img_train: {len(cat_img_train)}\")\nprint(f\"Dog_img_train: {len(dog_img_train)}\")\nprint(f\"Total_img_train: {len(file_list_train)}\")\n\n# val ディレクトリの中身確認\ncat_img_val = val_path / 'cat'\ndog_img_val = val_path / 'dog'\n\nif not cat_img_val.exists() or len(list(cat_img_val.iterdir())) == 0:\n    print(\"val/cat フォルダには画像が入っていません\")\nelse:\n    print(f\"val/cat フォルダに {len(list(cat_img_val.iterdir()))} 枚の画像があります\")\n\nif not dog_img_val.exists() or len(list(dog_img_val.iterdir())) == 0:\n    print(\"val/dog フォルダには画像が入っていません\")\nelse:\n    print(f\"val/dog フォルダに {len(list(dog_img_val.iterdir()))} 枚の画像があります\")\n\n# test データ確認\ntest_files = list(test_path.iterdir())\nprint(f\"Total_img_test: {len(test_files)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:53:57.934538Z","iopub.execute_input":"2025-07-25T05:53:57.934973Z","iopub.status.idle":"2025-07-25T05:54:11.519188Z","shell.execute_reply.started":"2025-07-25T05:53:57.934947Z","shell.execute_reply":"2025-07-25T05:54:11.518574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 不良画像の削除","metadata":{}},{"cell_type":"code","source":"\n# 不良画像リスト\nbad_images = [\n    'dog.10797.jpg', 'dog.10747.jpg', 'dog.10237.jpg', 'dog.9517.jpg',\n    'dog.8736.jpg', 'dog.5604.jpg', 'dog.1043.jpg', 'cat.4338.jpg',\n    'dog.10161.jpg', 'dog.10190.jpg'\n]\n\n# train_path は先ほどのコードで定義した Path オブジェクト\nremoved_count = 0\nfor img in bad_images:\n    img_path = train_path / img\n    if img_path.exists():\n        img_path.unlink()  # ファイル削除\n        removed_count += 1\n        print(f\"削除しました: {img}\")\n    else:\n        print(f\"見つかりませんでした: {img}\")\n\nprint(f\"\\n合計 {removed_count} 件の不良画像を削除しました\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:54:11.520905Z","iopub.execute_input":"2025-07-25T05:54:11.521367Z","iopub.status.idle":"2025-07-25T05:54:11.527161Z","shell.execute_reply.started":"2025-07-25T05:54:11.521348Z","shell.execute_reply":"2025-07-25T05:54:11.526418Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データローダーの作成と確認","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedShuffleSplit\n\n# 1. データ一覧とラベルを準備\nfiles = get_image_files(train_path)\nlabels = [1 if f.name.startswith('dog') else 0 for f in files]  # 0=cat,1=dog\n\n# 2. StratifiedSplitでインデックスを分割\nsplitter = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\ntrain_idx, valid_idx = next(splitter.split(files, labels))\n\n# 3. DataBlockを定義\ndblock = DataBlock(\n    blocks=(ImageBlock, CategoryBlock),\n    get_items=lambda _: files,\n    splitter=IndexSplitter(valid_idx),\n    get_y=lambda o: 'dog' if o.name.startswith('dog') else 'cat',\n    item_tfms=Resize(256),\n    batch_tfms=aug_transforms()\n)\n\n# 4. DataLoaders作成\ndls = dblock.dataloaders(source=None, bs=64)\n\n# 5. 確認（学習・検証の枚数とクラス分布）\nprint(f\"Train count: {len(dls.train_ds)}, Valid count: {len(dls.valid_ds)}\")\n\n# 6. ラベル分布の確認\nfrom collections import Counter\nprint(f\"Train labels: {Counter([dls.train_ds.items[i].name.startswith('dog') for i in range(len(dls.train_ds))])}\")\nprint(f\"Valid labels: {Counter([dls.valid_ds.items[i].name.startswith('dog') for i in range(len(dls.valid_ds))])}\")\n\n# 🔹 Trainデータの表示\nprint(\"🔍 Train データ（ラベル付き）\")\ndls.train.show_batch(max_n=9, figsize=(8,8), title='Train')\n\n# 🔹 Validationデータの表示\nprint(\"🔍 Validation データ（ラベル付き）\")\ndls.valid.show_batch(max_n=9, figsize=(8,8), title='Validation')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:54:11.527837Z","iopub.execute_input":"2025-07-25T05:54:11.528091Z","iopub.status.idle":"2025-07-25T05:54:15.404424Z","shell.execute_reply.started":"2025-07-25T05:54:11.528073Z","shell.execute_reply":"2025-07-25T05:54:15.403732Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データセットの前処理","metadata":{}},{"cell_type":"code","source":"# データ拡張の定義\nclass AlbTransform(Transform):\n    def __init__(self, aug): self.aug = aug\n    def encodes(self, img: PILImage):\n        aug_img = self.aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)\n    \ndef get_augs():\n    return Alb.Compose([\n        Alb.Affine(\n            rotate=(-20, 20),\n            translate_percent=(0, 0.1),\n            scale=(0.9, 1.1),\n            shear=0,\n            border_mode=0,   # 境界の埋め方（0は一定値で埋める）\n        ),\n        Alb.Transpose(),\n        Alb.HorizontalFlip(),\n        Alb.RandomRotate90(),\n        Alb.RandomBrightnessContrast(),\n        Alb.HueSaturationValue(hue_shift_limit=5, sat_shift_limit=5, val_shift_limit=5),\n        Alb.CoarseDropout()\n    ])\n\nitem_tfms = [Resize(256), AlbTransform(get_augs())] \nbatch_tfms = [Normalize.from_stats(*imagenet_stats), *aug_transforms()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:54:15.405425Z","iopub.execute_input":"2025-07-25T05:54:15.405677Z","iopub.status.idle":"2025-07-25T05:54:15.421290Z","shell.execute_reply.started":"2025-07-25T05:54:15.405657Z","shell.execute_reply":"2025-07-25T05:54:15.420561Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## モデルの作成","metadata":{}},{"cell_type":"markdown","source":"## ResNet101","metadata":{}},{"cell_type":"code","source":"# モデルの作成\nlearn = vision_learner(dls, resnet101, metrics=[error_rate, accuracy])\n\n#最適な学習率の推定\nlearn_RN101 = learn.lr_find()\nprint(learn_RN101) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:54:15.422053Z","iopub.execute_input":"2025-07-25T05:54:15.422301Z","iopub.status.idle":"2025-07-25T05:55:04.679585Z","shell.execute_reply.started":"2025-07-25T05:54:15.422285Z","shell.execute_reply":"2025-07-25T05:55:04.678594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 学習実行\nlearn.fit_one_cycle(5, lr_max=learn_RN101.valley)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T05:55:04.680624Z","iopub.execute_input":"2025-07-25T05:55:04.680968Z","iopub.status.idle":"2025-07-25T06:09:19.332465Z","shell.execute_reply.started":"2025-07-25T05:55:04.680933Z","shell.execute_reply":"2025-07-25T06:09:19.331610Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テストデータの準備","metadata":{}},{"cell_type":"code","source":"# テストデータの準備\nprint('Testing', len(test_files), 'items')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:09:19.333675Z","iopub.execute_input":"2025-07-25T06:09:19.334041Z","iopub.status.idle":"2025-07-25T06:09:19.338735Z","shell.execute_reply.started":"2025-07-25T06:09:19.334003Z","shell.execute_reply":"2025-07-25T06:09:19.338138Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テストデータセットローダーの作成","metadata":{}},{"cell_type":"code","source":"tst_dl = dls.test_dl(test_files, with_labels=False, shuffle=False, batch_tfms=None)\nbatch_tfms=[Normalize.from_stats(*imagenet_stats)]\ntst_dl.show_batch(max_n=12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:09:19.341153Z","iopub.execute_input":"2025-07-25T06:09:19.341405Z","iopub.status.idle":"2025-07-25T06:09:20.322090Z","shell.execute_reply.started":"2025-07-25T06:09:19.341388Z","shell.execute_reply":"2025-07-25T06:09:20.321293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# データ拡張なしの test_dl を明示的に作る\ntst_dl = dls.test_dl(test_files, with_labels=False, shuffle=False)\ntst_dl.show_batch(max_n=12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:09:20.322961Z","iopub.execute_input":"2025-07-25T06:09:20.323192Z","iopub.status.idle":"2025-07-25T06:09:21.243311Z","shell.execute_reply.started":"2025-07-25T06:09:20.323175Z","shell.execute_reply":"2025-07-25T06:09:21.242497Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 推論","metadata":{}},{"cell_type":"code","source":"# 推論実行\nstartTime = time.time()\nprobs, _ = learn.tta(dl=tst_dl, n=5, use_max=False)\nprint('TTA in:', time.time()-startTime, 'secs')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:09:21.244197Z","iopub.execute_input":"2025-07-25T06:09:21.244425Z","iopub.status.idle":"2025-07-25T06:13:44.571633Z","shell.execute_reply.started":"2025-07-25T06:09:21.244408Z","shell.execute_reply":"2025-07-25T06:13:44.570708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(learn.dls.vocab)       # 例: CategoryMap(['cat', 'dog'], sort=False)\nprint(learn.dls.vocab.o2i)   # 例: {'cat': 0, 'dog': 1}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:13:44.572894Z","iopub.execute_input":"2025-07-25T06:13:44.573225Z","iopub.status.idle":"2025-07-25T06:13:44.577938Z","shell.execute_reply.started":"2025-07-25T06:13:44.573189Z","shell.execute_reply":"2025-07-25T06:13:44.577234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# dogクラスのインデックス取得\ndog_idx = learn.dls.vocab.o2i['dog']\ndog_probs = probs[:, dog_idx]\n\n# ファイル名からID抽出\nids = [int(f.name.split('.')[0]) for f in tst_dl.items]\n\n# 提出用CSV作成\nimport pandas as pd\nsubmission = pd.DataFrame({'id': ids, 'label': dog_probs.numpy()})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:15:35.124625Z","iopub.execute_input":"2025-07-25T06:15:35.125271Z","iopub.status.idle":"2025-07-25T06:15:35.156898Z","shell.execute_reply.started":"2025-07-25T06:15:35.125246Z","shell.execute_reply":"2025-07-25T06:15:35.156292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 推論結果の表示","metadata":{}},{"cell_type":"code","source":"# ラベル取得（最大確率のインデックス）\npreds = probs.argmax(dim=1)\nclass_names = learn.dls.vocab\n# ランダムに12枚表示\nidxs = random.sample(range(len(preds)), 12)\nfig, axs = plt.subplots(3, 4, figsize=(12, 9))\n\nfor ax, idx in zip(axs.flatten(), idxs):\n    img = PILImage.create(tst_dl.items[idx])\n    pred_label = preds[idx].item()\n    pred_class = class_names[pred_label]\n    prob = probs[idx][pred_label].item()\n    ax.imshow(img)\n    ax.set_title(f'{pred_class} ({pred_label})\\nprob: {prob:.2f}')\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T06:13:44.635616Z","iopub.execute_input":"2025-07-25T06:13:44.636134Z","iopub.status.idle":"2025-07-25T06:13:45.699734Z","shell.execute_reply.started":"2025-07-25T06:13:44.636108Z","shell.execute_reply":"2025-07-25T06:13:45.698635Z"}},"outputs":[],"execution_count":null}]}