{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:38.670270Z","iopub.execute_input":"2025-07-11T19:24:38.670858Z","iopub.status.idle":"2025-07-11T19:24:38.676997Z","shell.execute_reply.started":"2025-07-11T19:24:38.670835Z","shell.execute_reply":"2025-07-11T19:24:38.676420Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport time\nfrom fastai.vision.all import *\nimport zipfile\nimport albumentations as Alb\nimport random\nimport shutil\nimport PIL\nimport torch \nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom fastai.vision.augment import RandTransform\nfrom albumentations.pytorch import ToTensorV2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:38.678335Z","iopub.execute_input":"2025-07-11T19:24:38.678804Z","iopub.status.idle":"2025-07-11T19:24:38.692824Z","shell.execute_reply.started":"2025-07-11T19:24:38.678786Z","shell.execute_reply":"2025-07-11T19:24:38.692197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## zipファイルの展開","metadata":{}},{"cell_type":"code","source":"#訓練データの解凍\nzip_path_train = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\ntemp_dir_train = tempfile.TemporaryDirectory()\nwith zipfile.ZipFile(zip_path_train, 'r') as zip_ref:\n    zip_ref.extractall(temp_dir_train.name)\ntrain_path = Path(temp_dir_train.name) / 'train'\nval_path = Path(temp_dir_train.name) / 'val'  # 今は使わなくてもOK\n\n# テストデータの解凍\nzip_path_test = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\ntemp_dir_test = tempfile.TemporaryDirectory()\nwith zipfile.ZipFile(zip_path_test, 'r') as zip_ref:\n    zip_ref.extractall(temp_dir_test.name)\ntest_path = Path(temp_dir_test.name) / 'test'\n\n# train フォルダ内ファイル一覧\nfile_list_train = list(train_path.iterdir())\n\n# 猫と犬のファイル名パターンでカウント\ncat_img_train = [f for f in file_list_train if f.name.startswith(\"cat\")]\ndog_img_train = [f for f in file_list_train if f.name.startswith(\"dog\")]\n\nprint(f\"Cat_img_train: {len(cat_img_train)}\")\nprint(f\"Dog_img_train: {len(dog_img_train)}\")\nprint(f\"Total_img_train: {len(file_list_train)}\")\n\n# val ディレクトリの中身確認\ncat_img_val = val_path / 'cat'\ndog_img_val = val_path / 'dog'\n\nif not cat_img_val.exists() or len(list(cat_img_val.iterdir())) == 0:\n    print(\"val/cat フォルダには画像が入っていません\")\nelse:\n    print(f\"val/cat フォルダに {len(list(cat_img_val.iterdir()))} 枚の画像があります\")\n\nif not dog_img_val.exists() or len(list(dog_img_val.iterdir())) == 0:\n    print(\"val/dog フォルダには画像が入っていません\")\nelse:\n    print(f\"val/dog フォルダに {len(list(dog_img_val.iterdir()))} 枚の画像があります\")\n\n# test データ確認\ntest_files = list(test_path.iterdir())\nprint(f\"Total_img_test: {len(test_files)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:38.693546Z","iopub.execute_input":"2025-07-11T19:24:38.693782Z","iopub.status.idle":"2025-07-11T19:24:48.498754Z","shell.execute_reply.started":"2025-07-11T19:24:38.693760Z","shell.execute_reply":"2025-07-11T19:24:48.497926Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 不良画像の削除","metadata":{}},{"cell_type":"code","source":"\n# 不良画像リスト\nbad_images = [\n    'dog.10797.jpg', 'dog.10747.jpg', 'dog.10237.jpg', 'dog.9517.jpg',\n    'dog.8736.jpg', 'dog.5604.jpg', 'dog.1043.jpg', 'cat.4338.jpg',\n    'dog.10161.jpg', 'dog.10190.jpg'\n]\n\n# train_path は先ほどのコードで定義した Path オブジェクト\nremoved_count = 0\nfor img in bad_images:\n    img_path = train_path / img\n    if img_path.exists():\n        img_path.unlink()  # ファイル削除\n        removed_count += 1\n        print(f\"削除しました: {img}\")\n    else:\n        print(f\"見つかりませんでした: {img}\")\n\nprint(f\"\\n合計 {removed_count} 件の不良画像を削除しました\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:48.499564Z","iopub.execute_input":"2025-07-11T19:24:48.499923Z","iopub.status.idle":"2025-07-11T19:24:48.523034Z","shell.execute_reply.started":"2025-07-11T19:24:48.499903Z","shell.execute_reply":"2025-07-11T19:24:48.522321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データローダーの作成と確認","metadata":{}},{"cell_type":"code","source":"# 1. データ一覧とラベルを準備\nfiles = get_image_files(train_path)\nlabels = [1 if f.name.startswith('dog') else 0 for f in files]  # 0=cat,1=dog\n\n# 2. StratifiedSplitでインデックスを分割\nsplitter = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\ntrain_idx, valid_idx = next(splitter.split(files, labels))\n\n# 3. DataBlockを定義\ndblock = DataBlock(\n    blocks=(ImageBlock, CategoryBlock),\n    get_items=lambda _: files,\n    splitter=IndexSplitter(valid_idx),\n    get_y=lambda o: 'dog' if o.name.startswith('dog') else 'cat',\n    item_tfms=Resize(256),\n    batch_tfms=aug_transforms()\n)\n\n# 4. DataLoaders作成\ndls = dblock.dataloaders(source=None, bs=64)\n\n# 5. 確認（学習・検証の枚数とクラス分布）\nprint(f\"Train count: {len(dls.train_ds)}, Valid count: {len(dls.valid_ds)}\")\n\n# 6. ラベル分布の確認\nfrom collections import Counter\nprint(f\"Train labels: {Counter([dls.train_ds.items[i].name.startswith('dog') for i in range(len(dls.train_ds))])}\")\nprint(f\"Valid labels: {Counter([dls.valid_ds.items[i].name.startswith('dog') for i in range(len(dls.valid_ds))])}\")\n\n# 🔹 Trainデータの表示\nprint(\"🔍 Train データ（ラベル付き）\")\ndls.train.show_batch(max_n=9, figsize=(8,8), title='Train')\n\n# 🔹 Validationデータの表示\nprint(\"🔍 Validation データ（ラベル付き）\")\ndls.valid.show_batch(max_n=9, figsize=(8,8), title='Validation')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:48.524956Z","iopub.execute_input":"2025-07-11T19:24:48.525192Z","iopub.status.idle":"2025-07-11T19:24:52.858115Z","shell.execute_reply.started":"2025-07-11T19:24:48.525169Z","shell.execute_reply":"2025-07-11T19:24:52.857178Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データセットの前処理","metadata":{}},{"cell_type":"code","source":"# データ拡張の定義\nclass AlbTransform(Transform):\n    def __init__(self, aug): self.aug = aug\n    def encodes(self, img: PILImage):\n        aug_img = self.aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)\n    \ndef get_augs():\n    return Alb.Compose([\n        Alb.Affine(\n            rotate=(-20, 20),\n            translate_percent=(0, 0.1),\n            scale=(0.9, 1.1),\n            shear=0,\n            border_mode=0,   # 境界の埋め方（0は一定値で埋める）\n        ),\n        Alb.Transpose(),\n        Alb.HorizontalFlip(),\n        Alb.RandomRotate90(),\n        Alb.RandomBrightnessContrast(),\n        Alb.HueSaturationValue(hue_shift_limit=5, sat_shift_limit=5, val_shift_limit=5),\n        Alb.CoarseDropout()\n    ])\n\nitem_tfms = [Resize(256), AlbTransform(get_augs())] \nbatch_tfms = [Normalize.from_stats(*imagenet_stats), *aug_transforms()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:52.858950Z","iopub.execute_input":"2025-07-11T19:24:52.859190Z","iopub.status.idle":"2025-07-11T19:24:52.870554Z","shell.execute_reply.started":"2025-07-11T19:24:52.859165Z","shell.execute_reply":"2025-07-11T19:24:52.869863Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## モデルの作成","metadata":{}},{"cell_type":"markdown","source":"## ResNet101","metadata":{}},{"cell_type":"code","source":"# モデルの作成\nlearn = vision_learner(dls, resnet101, metrics=[error_rate, accuracy])\n\n#最適な学習率の推定\nlearn_RN101 = learn.lr_find()\nprint(learn_RN101) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:24:52.871484Z","iopub.execute_input":"2025-07-11T19:24:52.871776Z","iopub.status.idle":"2025-07-11T19:25:40.443186Z","shell.execute_reply.started":"2025-07-11T19:24:52.871746Z","shell.execute_reply":"2025-07-11T19:25:40.442384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 学習実行\nlearn.fit_one_cycle(5, lr_max=learn_RN101.valley)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:25:40.444080Z","iopub.execute_input":"2025-07-11T19:25:40.444362Z","iopub.status.idle":"2025-07-11T19:39:59.555297Z","shell.execute_reply.started":"2025-07-11T19:25:40.444340Z","shell.execute_reply":"2025-07-11T19:39:59.554543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テストデータの準備","metadata":{}},{"cell_type":"code","source":"# テストデータの準備\nprint('Testing', len(test_files), 'items')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:39:59.556563Z","iopub.execute_input":"2025-07-11T19:39:59.557286Z","iopub.status.idle":"2025-07-11T19:39:59.563856Z","shell.execute_reply.started":"2025-07-11T19:39:59.557250Z","shell.execute_reply":"2025-07-11T19:39:59.562124Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テストデータセットローダーの作成","metadata":{}},{"cell_type":"code","source":"# データ拡張なしの test_dl を明示的に作る\ntst_dl = dls.test_dl(test_files, with_labels=False, shuffle=False)\ntst_dl.show_batch(max_n=12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:40:00.542763Z","iopub.execute_input":"2025-07-11T19:40:00.543035Z","iopub.status.idle":"2025-07-11T19:40:01.484540Z","shell.execute_reply.started":"2025-07-11T19:40:00.543011Z","shell.execute_reply":"2025-07-11T19:40:01.483503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 推論","metadata":{}},{"cell_type":"code","source":"# 推論実行\nstartTime = time.time()\nprobs, _ = learn.tta(dl=tst_dl, n=5, use_max=False)\nprint('TTA in:', time.time()-startTime, 'secs')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:40:01.485510Z","iopub.execute_input":"2025-07-11T19:40:01.485718Z","iopub.status.idle":"2025-07-11T19:44:25.389228Z","shell.execute_reply.started":"2025-07-11T19:40:01.485703Z","shell.execute_reply":"2025-07-11T19:44:25.388206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(learn.dls.vocab)       # 例: CategoryMap(['cat', 'dog'], sort=False)\nprint(learn.dls.vocab.o2i)   # 例: {'cat': 0, 'dog': 1}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:44:25.390423Z","iopub.execute_input":"2025-07-11T19:44:25.390678Z","iopub.status.idle":"2025-07-11T19:44:25.396448Z","shell.execute_reply.started":"2025-07-11T19:44:25.390647Z","shell.execute_reply":"2025-07-11T19:44:25.395553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# dogクラスのインデックス取得\ndog_idx = learn.dls.vocab.o2i['dog']\ndog_probs = probs[:, dog_idx]\n\n# ファイル名からID抽出\nids = [int(f.name.split('.')[0]) for f in tst_dl.items]\n\n# 提出用CSV作成\nimport pandas as pd\nsubmission = pd.DataFrame({'id': ids, 'label': dog_probs.numpy()})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:44:25.399477Z","iopub.execute_input":"2025-07-11T19:44:25.399941Z","iopub.status.idle":"2025-07-11T19:44:25.455724Z","shell.execute_reply.started":"2025-07-11T19:44:25.399922Z","shell.execute_reply":"2025-07-11T19:44:25.455164Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 推論結果の表示","metadata":{}},{"cell_type":"code","source":"# ラベル取得（最大確率のインデックス）\npreds = probs.argmax(dim=1)\nclass_names = learn.dls.vocab\n# ランダムに12枚表示\nidxs = random.sample(range(len(preds)), 12)\nfig, axs = plt.subplots(3, 4, figsize=(12, 9))\n\nfor ax, idx in zip(axs.flatten(), idxs):\n    img = PILImage.create(tst_dl.items[idx])\n    pred_label = preds[idx].item()\n    pred_class = class_names[pred_label]\n    prob = probs[idx][pred_label].item()\n    ax.imshow(img)\n    ax.set_title(f'{pred_class} ({pred_label})\\nprob: {prob:.2f}')\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T19:44:25.456495Z","iopub.execute_input":"2025-07-11T19:44:25.456766Z","iopub.status.idle":"2025-07-11T19:44:26.582460Z","shell.execute_reply.started":"2025-07-11T19:44:25.456743Z","shell.execute_reply":"2025-07-11T19:44:26.581228Z"}},"outputs":[],"execution_count":null}]}