{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"},{"sourceId":3362,"databundleVersionId":31148,"sourceType":"competition"},{"sourceId":200252001,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ls /kaggle/input/notebookc785dff359/dogs-vs-cats-redux-kernels-edition/","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-17T05:02:29.218859Z","iopub.execute_input":"2024-10-17T05:02:29.219150Z","iopub.status.idle":"2024-10-17T05:02:30.225749Z","shell.execute_reply.started":"2024-10-17T05:02:29.219115Z","shell.execute_reply":"2024-10-17T05:02:30.224691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport shutil\nimport torchvision.transforms as transforms\nfrom typing import List, Tuple\n\nimport os\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport torch\nfrom torch import nn, optim\nfrom torch.optim.lr_scheduler import StepLR\n\nfrom torch.utils.data import DataLoader, Dataset\nimport numpy as np\nimport pickle\n\nimport zipfile\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nBATCH_SIZE = 256\nsource_filename = \"dogs-vs-cats-redux-kernels-edition\"\n# dataset_dir = Path(\"H:/dataset\")\ndataset_dir = Path(\"/kaggle/input/notebookc785dff359\")\n\n\ndef _unzip(base_file_path: Path, target_filename, destination=None):\n    if target_filename[-4:] == \".zip\":\n        target_filename = target_filename[:-4]\n    source_zip_path = base_file_path / (target_filename + \".zip\")\n    if destination is not None:\n        source_path = destination / target_filename\n    else:\n        source_path = base_file_path / target_filename\n\n    if not os.path.exists(source_zip_path):\n        raise Exception(\n            \"加藤より指定されたzipが正しい位置に存在しないので、ダウンロードしてください。わからなかったら加藤に聞いて\"\n        )\n\n    # zipからデータを読み出して、拡張子を変更して保存\n    if not os.path.exists(source_path):\n        with zipfile.ZipFile(source_zip_path, \"r\") as zip_ref:\n            zip_ref.extractall(\n                source_path,\n            )\n            zip_ref.close()\n\n    print(f\"{source_zip_path} を解凍しました。解凍先：{source_path}\")\n    return source_path\n\n\nclass CatsDogsDataset(Dataset):\n    def __init__(self, path, transform=None):\n        self.data = []\n        self.transform = transform\n        self.label_map = {\"cat\": [1.0, 0.0], \"dog\": [0.0, 1.0]}\n        self.max_width, self.max_height = (\n            1200,\n            1000,\n        )  # Example dimensions, adjust as needed\n        self.pickle_dir = path.parent / (path.stem + \"_pickle\")\n        self.pickle_dir.mkdir(exist_ok=True)\n        for file in tqdm(os.listdir(path), desc=\"Processing files\"):\n            if file.endswith(\".jpg\"):\n                img_path = path / file\n                label = \"cat\" if \"cat\" in file else \"dog\"\n                pickle_path = self.pickle_dir / f\"{file}.pkl\"\n\n                if not pickle_path.exists():\n                    image = self._load_image(img_path)\n                    with open(pickle_path, \"wb\") as f:\n                        pickle.dump((image, self.label_map[label]), f)\n\n                self.data.append(pickle_path)\n                # self.data.append((img_path, self.label_map[label]))\n\n    def _pad_image(self, image, max_width, max_height):\n        width, height = image.size\n        new_image = Image.new(\"RGB\", (max_width, max_height))\n        new_image.paste(image, ((max_width - width) // 2, (max_height - height) // 2))\n        return new_image\n\n    def _load_image(self, path: Path):\n        img = Image.open(path).convert(\"RGB\")\n        img = self._pad_image(img, self.max_width, self.max_height)\n        img = img.resize((120, 100))\n        # img = transforms.ToTensor()(img)  # Convert PIL Image to Tensor\n        img = np.array(img).astype(np.float32)\n        img = img / 255.0  # Normalize pixel values to [0, 1\n        # 次元をchannelfirstにしたい\n        img = np.transpose(img, (2, 0, 1))  # Change dimensions to (C, H, W)\n\n        return img\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        pickle_path = self.data[idx]\n        with open(pickle_path, \"rb\") as f:\n            image, label = pickle.load(f)\n        # image = self._load_image(pickle_path)\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image.astype(np.float32), torch.tensor(label)\n\n\ndef create_data():\n#     data_path = _unzip(dataset_dir, source_filename)\n    data_path = Path(\"/kaggle/input/notebookc785dff359/dogs-vs-cats-redux-kernels-edition/\")\n#     test_path =_unzip(data_path, \"test.zip\",destination=Path(\"/kaggle/working/dogs-vs-cats-redux-kernels-edition\")) / \"test\"\n#     train_path = _unzip(data_path, \"train.zip\",destination=Path(\"/kaggle/working/dogs-vs-cats-redux-kernels-edition\")) / 'train'\n    \n    test_path = data_path / \"test\" / \"test\"\n    train_path = data_path / \"train\" / \"train\"\n\n    # データセットのインスタンスを作成\n    train_dataset = CatsDogsDataset(train_path)\n    test_dataset = CatsDogsDataset(test_path)\n\n    # DataLoaderを作成\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    return train_loader, test_loader\n\n\n# def _calc_accuracy_point(prediction, y) -> torch.Tensor:\n#     diff = torch.abs(prediction - y)\n#     diff = diff.mean()\n#     accuracy_point = 1 - diff / y.mean()\n#     return accuracy_point\n\n\ndef _calc_accuracy_point(prediction, y) -> torch.Tensor:\n    assert len(prediction.shape) == 2\n    assert len(y.shape) == 2\n    correct = torch.sum(torch.argmax(prediction, -1) == torch.argmax(y, -1))\n    total = y.size(0)\n\n    # 正解率を計算\n    accuracy = correct / total\n    return accuracy\n\n\n# def _calc_accuracy_point(prediction, y) -> torch.Tensor:\n#     correct = (prediction == y).sum()\n#     total = y.size(0)\n\n#     # 正解率を計算\n#     accuracy = (correct / total) * 100\n#     return accuracy\n\n\ndef _visualize(x, y):\n    return nothing\n\n\nclass Criterion(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.criterion = nn.CrossEntropyLoss()\n\n    def forward(self, pred, y):\n        # yは正解ラベルの数値なので、one-hotエンコーディングをしたい\n        return self.criterion(pred, y)\n\n\ndef train(model, data, epochs, lr=0.01, criterion=Criterion(), lr_scheduler=StepLR):\n    criterion = criterion.to(device)\n    model = model.to(device)\n    # 損失関数と最適化手法を定義\n    optimizer = optim.Adam(model.parameters(), lr=lr)\n    if lr_scheduler is not None:\n        scheduler = lr_scheduler(optimizer, step_size=1, gamma=0.8)\n\n    step_losses = []\n    for epoch in range(epochs):\n        with tqdm(enumerate(data), total=len(data)) as pbar:\n            pbar.set_description(f\"[Epoch {epoch + 1}/{epochs}]\")\n\n            loss_list = []\n            accuracy_list = []\n\n            for batch_idx, (x, y) in pbar:\n                # 順伝播\n                x = x.to(device)\n                y = y.to(device)\n                outputs = model(x)\n                loss = criterion(outputs, y)\n\n                # 逆伝播と最適化\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                step_losses.append(loss.item())\n                accuracy = _calc_accuracy_point(outputs, y)\n\n                loss_list.append(loss.item())\n                accuracy_list.append(accuracy.item())\n\n                pbar.set_postfix(\n                    OrderedDict(\n                        Loss=loss.item(),\n                        acc=accuracy.item(),\n                        ave_ac=np.mean(accuracy_list),\n                        ave_loss=np.mean(loss_list),\n                        lr=optimizer.param_groups[0][\"lr\"],\n                    )\n                )\n\n            if lr_scheduler is not None:\n                scheduler.step()\n\n            if epoch % 100 == 0:\n                print(\n                    f\"Train Epoch: {epoch} [{batch_idx * len(x)}/{len(data.dataset)} \"\n                    f\"({100. * batch_idx / len(data):.0f}%)]\\tLoss: {loss.item():.6f}\"\n                )\n\n    # 損失のプロット\n    plt.plot(step_losses)\n    plt.xlabel(\"Step\")\n    plt.ylabel(\"Loss\")\n    plt.title(\"Training Loss over Time\")\n    plt.show()\n\n\ndef visualize(model, data, data_count):\n    model = model.to(device)\n\n    count = 0\n\n    finish_flag = False\n\n    for batch_idx, (batch_x, batch_y) in enumerate(data):\n        for x, y in zip(batch_x, batch_y):\n            _visualize(x, y)\n\n            if count >= data_count:\n                finish_flag = True\n                break\n        if finish_flag:\n            break\n\n\ndef test(model, data):\n    model = model.to(device)\n\n    accuracy_amount = 0\n\n    for batch_idx, (x, y) in enumerate(data):\n        x = x.to(device)\n        y = y.to(device)\n        prediction = model(x)\n        accuracy_amount += _calc_accuracy_point(prediction, y)\n\n    accuracy = accuracy_amount / len(data)\n\n    print(f\"Accuracy: {accuracy*100}%\")\n\n\nprint(f\"Device:{device}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:02:30.227948Z","iopub.execute_input":"2024-10-17T05:02:30.228285Z","iopub.status.idle":"2024-10-17T05:02:35.090242Z","shell.execute_reply.started":"2024-10-17T05:02:30.228248Z","shell.execute_reply":"2024-10-17T05:02:35.089277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 犬と猫の画像判定\n\n例えば、[0.5,0.5]を出力。前者が犬っぽい確率。後者は猫っぽい確率\n","metadata":{}},{"cell_type":"code","source":"train_data, test_data = create_data()","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:02:35.091450Z","iopub.execute_input":"2024-10-17T05:02:35.091946Z","iopub.status.idle":"2024-10-17T05:03:41.522137Z","shell.execute_reply.started":"2024-10-17T05:02:35.091909Z","shell.execute_reply":"2024-10-17T05:03:41.521226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x,y in train_data:\n    print(x.shape)\n    print(y.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:03:41.524528Z","iopub.execute_input":"2024-10-17T05:03:41.524876Z","iopub.status.idle":"2024-10-17T05:03:42.894190Z","shell.execute_reply.started":"2024-10-17T05:03:41.524840Z","shell.execute_reply":"2024-10-17T05:03:42.893088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 以下にmodelを定義してみよう。\n\n※そのまえに、必ず上のすべてのプログラムをまず実行させてみてね？\n\nmodel =","metadata":{}},{"cell_type":"code","source":"from torchvision import models\n# pretrained=Trueと同等のモデルを呼び出したい！\n# 1. 明示的に重みを指定\nmodel_base = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1)\n\n# model_base(train_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:03:42.895397Z","iopub.execute_input":"2024-10-17T05:03:42.895758Z","iopub.status.idle":"2024-10-17T05:03:44.030105Z","shell.execute_reply.started":"2024-10-17T05:03:42.895702Z","shell.execute_reply":"2024-10-17T05:03:44.029089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = nn.Sequential(\nmodel_base,\nnn.Linear(1000,128),\nnn.ReLU(),\nnn.Linear(128,2),\nnn.Softmax(-1))","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:08:26.928541Z","iopub.execute_input":"2024-10-17T05:08:26.929418Z","iopub.status.idle":"2024-10-17T05:08:26.936634Z","shell.execute_reply.started":"2024-10-17T05:08:26.929361Z","shell.execute_reply":"2024-10-17T05:08:26.935702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_epochs = 20\n\ntrain(model,train_data,number_of_epochs)\ntest(model,test_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:10:13.709015Z","iopub.execute_input":"2024-10-17T05:10:13.709829Z","iopub.status.idle":"2024-10-17T05:13:05.643812Z","shell.execute_reply.started":"2024-10-17T05:10:13.709785Z","shell.execute_reply":"2024-10-17T05:13:05.642769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_epochs = 20\ntrain(model,train_data,number_of_epochs,0.01)\ntest(model,test_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:08:20.882197Z","iopub.status.idle":"2024-10-17T05:08:20.882539Z","shell.execute_reply.started":"2024-10-17T05:08:20.882370Z","shell.execute_reply":"2024-10-17T05:08:20.882388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"もしモデルが完成したら、以下のコードを実行してください。すると、submit.csvができる。それを提出すると成績が出ますや","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip .\n!ls\n_unzip(Path(\".\"),\"test.zip\")\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()\n\ndef _pad_image(image, max_width, max_height):\n    width, height = image.size\n    new_image = Image.new(\"RGB\", (max_width, max_height))\n    new_image.paste(image, ((max_width - width) // 2, (max_height - height) // 2))\n    return new_image\n\nmax_width, max_height = (\n            1200,\n            1000,\n        )  # Example di\n\ndef _load_image(path: Path):\n    img = Image.open(path).convert(\"RGB\")\n    img = _pad_image(img, max_width, max_height)\n    img = img.resize((120, 100))\n    # img = transforms.ToTensor()(img)  # Convert PIL Image to Tensor\n    img = np.array(img).astype(np.float32)\n    img = img / 255.0  # Normalize pixel values to [0, 1\n    # 次元をchannelfirstにしたい\n    img = np.transpose(img, (2, 0, 1))  # Change dimensions to (C, H, W)\n\n    return torch.tensor(img,dtype=torch.float32,device='cuda').unsqueeze(0)\n\n\nresults = []\ncount = 0\nimgs = []\nids = []\nlength = len(os.listdir('test/test'))\nwith torch.no_grad():\n    for file in os.listdir(\"test/test\"):\n        img = _load_image(\"test/test/\"+file)\n        imgs.append(img)\n#         predict = model(img)[0][0]\n\n        id = int(file.split(\".\")[0])\n        ids.append(id)\n\n        if count%16384 ==0 or count :\n#             print(count/length)\n            imgs = torch.cat(imgs,dim=0)\n            predicts = model(imgs)\n            for id,predict in zip(ids,predicts):\n                results.append([int(id),predict])\n            imgs = []\n            ids = []\n            \n            \n        count+=1\n    if len(imgs)!=0:\n        imgs = torch.cat(imgs,dim=0)\n        predicts = model(imgs)\n        for id,predict in zip(ids,predicts):\n            results.append([int(id),predict])\n        imgs = []\n        ids = []\n\n    \nresults = [[result[0],int(result[1].argmax(-1))] for result in results]\n\nimport pandas as pd\nimport numpy as np\ndf = pd.DataFrame(np.array(results),columns=['id','label'])\n\ndf.to_csv(\"submit.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-17T05:16:08.481189Z","iopub.execute_input":"2024-10-17T05:16:08.481630Z","iopub.status.idle":"2024-10-17T05:20:13.657538Z","shell.execute_reply.started":"2024-10-17T05:16:08.481589Z","shell.execute_reply":"2024-10-17T05:20:13.656663Z"},"trusted":true},"execution_count":null,"outputs":[]}]}