{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"},{"sourceId":200252001,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ls /kaggle/input/notebookc785dff359/dogs-vs-cats-redux-kernels-edition/","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-24T06:23:38.007630Z","iopub.execute_input":"2024-10-24T06:23:38.008033Z","iopub.status.idle":"2024-10-24T06:23:39.035654Z","shell.execute_reply.started":"2024-10-24T06:23:38.007994Z","shell.execute_reply":"2024-10-24T06:23:39.034507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport random\nfrom PIL import Image\nimport shutil\nimport torchvision.transforms as transforms\nfrom typing import List, Tuple\n\nimport os\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport torch\nfrom torch import nn, optim\nfrom torch.optim.lr_scheduler import StepLR\n\nfrom torch.utils.data import DataLoader, Dataset\nimport numpy as np\nimport pickle\n\nimport zipfile\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nBATCH_SIZE = 256\nsource_filename = \"dogs-vs-cats-redux-kernels-edition\"\n# dataset_dir = Path(\"H:/dataset\")\ndataset_dir = Path(\"/kaggle/input/notebookc785dff359\")\n\n\ndef _unzip(base_file_path: Path, target_filename, destination=None):\n    if target_filename[-4:] == \".zip\":\n        target_filename = target_filename[:-4]\n    source_zip_path = base_file_path / (target_filename + \".zip\")\n    if destination is not None:\n        source_path = destination / target_filename\n    else:\n        source_path = base_file_path / target_filename\n\n    if not os.path.exists(source_zip_path):\n        raise Exception(\n            \"加藤より指定されたzipが正しい位置に存在しないので、ダウンロードしてください。わからなかったら加藤に聞いて\"\n        )\n\n    # zipからデータを読み出して、拡張子を変更して保存\n    if not os.path.exists(source_path):\n        with zipfile.ZipFile(source_zip_path, \"r\") as zip_ref:\n            zip_ref.extractall(\n                source_path,\n            )\n            zip_ref.close()\n\n    print(f\"{source_zip_path} を解凍しました。解凍先：{source_path}\")\n    return source_path\n    \nclass CatsDogsDataset(Dataset):\n    def __init__(self, path, transform=None):\n        self.data = []\n        self.transform = transform\n        self.label_map = {\"cat\": [1.0, 0.0], \"dog\": [0.0, 1.0]}\n        self.max_width, self.max_height = (\n            1200,\n            1000,\n        )  # Example dimensions, adjust as needed\n        self.pickle_dir = path.parent / (path.stem + \"_pickle\")\n        self.pickle_dir.mkdir(exist_ok=True)\n        for file in tqdm(os.listdir(path), desc=\"Processing files\"):\n            if file.endswith(\".jpg\"):\n                img_path = path / file\n                label = \"cat\" if \"cat\" in file else \"dog\"\n                pickle_path = self.pickle_dir / f\"{file}.pkl\"\n\n                if not pickle_path.exists():\n                    image = self._load_image(img_path)\n                    with open(pickle_path, \"wb\") as f:\n                        pickle.dump((image, self.label_map[label]), f)\n\n                self.data.append(pickle_path)\n                # self.data.append((img_path, self.label_map[label]))\n\n    def _pad_image(self, image, max_width, max_height):\n        width, height = image.size\n        new_image = Image.new(\"RGB\", (max_width, max_height))\n        new_image.paste(image, ((max_width - width) // 2, (max_height - height) // 2))\n        return new_image\n\n    def _load_image(self, path: Path):\n        img = Image.open(path).convert(\"RGB\")\n        img = self._pad_image(img, self.max_width, self.max_height)\n        img = img.resize((240, 200))\n        # img = transforms.ToTensor()(img)  # Convert PIL Image to Tensor\n        img = np.array(img).astype(np.float32)\n        img = img / 255.0  # Normalize pixel values to [0, 1\n        # 次元をchannelfirstにしたい\n        img = np.transpose(img, (2, 0, 1))  # Change dimensions to (C, H, W)\n\n        return img\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        pickle_path = self.data[idx]\n        with open(pickle_path, \"rb\") as f:\n            image, label = pickle.load(f)\n        # image = self._load_image(pickle_path)\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image.astype(np.float32), torch.tensor(label)\n\n\ndef create_data():\n#     data_path = _unzip(dataset_dir, source_filename)\n    data_path = Path(\"/kaggle/input/notebookc785dff359/dogs-vs-cats-redux-kernels-edition/\")\n#     test_path =_unzip(data_path, \"test.zip\",destination=Path(\"/kaggle/working/dogs-vs-cats-redux-kernels-edition\")) / \"test\"\n#     train_path = _unzip(data_path, \"train.zip\",destination=Path(\"/kaggle/working/dogs-vs-cats-redux-kernels-edition\")) / 'train'\n\n    \n    test_path = data_path / \"test\" / \"test\"\n    train_path = data_path / \"train\" / \"train\"\n    \n\n    # データセットのインスタンスを作成\n    train_and_val_dataset = CatsDogsDataset(train_path)\n    test_dataset = CatsDogsDataset(test_path)\n    \n    length = len(train_and_val_dataset)\n    train_len = int(length*0.9)\n    val_len = length-train_len\n    train_dataset,val_dataset = torch.utils.data.random_split(train_and_val_dataset,[train_len,val_len])\n\n    # DataLoaderを作成\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    return train_loader,val_loader, test_loader\n\n\n# def _calc_accuracy_point(prediction, y) -> torch.Tensor:\n#     diff = torch.abs(prediction - y)\n#     diff = diff.mean()\n#     accuracy_point = 1 - diff / y.mean()\n#     return accuracy_point\n\n\ndef _calc_accuracy_point(prediction, y) -> torch.Tensor:\n    assert len(prediction.shape) == 2\n    assert len(y.shape) == 2\n    correct = torch.sum(torch.argmax(prediction, -1) == torch.argmax(y, -1))\n    total = y.size(0)\n\n    # 正解率を計算\n    accuracy = correct / total\n    return accuracy\n\n\n# def _calc_accuracy_point(prediction, y) -> torch.Tensor:\n#     correct = (prediction == y).sum()\n#     total = y.size(0)\n\n#     # 正解率を計算\n#     accuracy = (correct / total) * 100\n#     return accuracy\n\n\ndef _visualize(x, y):\n    return nothing\n\n\nclass Criterion(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.criterion = nn.CrossEntropyLoss()\n\n    def forward(self, pred, y):\n        # yは正解ラベルの数値なので、one-hotエンコーディングをしたい\n        return self.criterion(pred, y)\n\n\ndef train(model, data,val_data, epochs, lr=0.01, criterion=Criterion(), lr_scheduler=StepLR):\n    criterion = criterion.to(device)\n    model = model.to(device)\n    # 損失関数と最適化手法を定義\n    optimizer = optim.Adam(model.parameters(), lr=lr)\n    if lr_scheduler is not None:\n        scheduler = lr_scheduler(optimizer, step_size=1, gamma=0.8)\n\n    step_losses = []\n    for epoch in range(epochs):\n        with tqdm(enumerate(data), total=len(data)) as pbar:\n            pbar.set_description(f\"[Epoch {epoch + 1}/{epochs}]\")\n\n            loss_list = []\n            accuracy_list = []\n\n            for batch_idx, (x, y) in pbar:\n                # 順伝播\n                x = x.to(device)\n                y = y.to(device)\n                outputs = model(x)\n                loss = criterion(outputs, y)\n\n                # 逆伝播と最適化\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                step_losses.append(loss.item())\n                accuracy = _calc_accuracy_point(outputs, y)\n\n                loss_list.append(loss.item())\n                accuracy_list.append(accuracy.item())\n\n                pbar.set_postfix(\n                    OrderedDict(\n                        Loss=loss.item(),\n                        acc=accuracy.item(),\n                        ave_ac=np.mean(accuracy_list),\n                        ave_loss=np.mean(loss_list),\n                        lr=optimizer.param_groups[0][\"lr\"],\n                    )\n                )\n                del x\n                del y\n            gc.collect()\n            torch.cuda.empty_cache()\n\n            if lr_scheduler is not None:\n                scheduler.step()\n            test(model,val_data)\n\n    # 損失のプロット\n    plt.plot(step_losses)\n    plt.xlabel(\"Step\")\n    plt.ylabel(\"Loss\")\n    plt.title(\"Training Loss over Time\")\n    plt.show()\n\n\ndef visualize(model, data, data_count):\n    model = model.to(device)\n\n    count = 0\n\n    finish_flag = False\n\n    for batch_idx, (batch_x, batch_y) in enumerate(data):\n        for x, y in zip(batch_x, batch_y):\n            _visualize(x, y)\n\n            if count >= data_count:\n                finish_flag = True\n                break\n        if finish_flag:\n            break\n\n\ndef test(model, data):\n    model = model.to(device)\n\n    accuracy_amount = 0\n\n    for batch_idx, (x, y) in enumerate(data):\n        x = x.to(device)\n        y = y.to(device)\n        prediction = model(x)\n        accuracy_amount += _calc_accuracy_point(prediction, y)\n\n    accuracy = accuracy_amount / len(data)\n\n    print(f\"Accuracy: {accuracy*100}%\")\n\n\nprint(f\"Device:{device}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-24T06:23:39.038441Z","iopub.execute_input":"2024-10-24T06:23:39.038888Z","iopub.status.idle":"2024-10-24T06:23:43.977285Z","shell.execute_reply.started":"2024-10-24T06:23:39.038826Z","shell.execute_reply":"2024-10-24T06:23:43.976232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 犬と猫の画像判定\n\n例えば、[0.5,0.5]を出力。前者が犬っぽい確率。後者は猫っぽい確率\n","metadata":{}},{"cell_type":"code","source":"train_data,val_data, test_data = create_data()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T06:23:43.978445Z","iopub.execute_input":"2024-10-24T06:23:43.978952Z","iopub.status.idle":"2024-10-24T06:24:38.399117Z","shell.execute_reply.started":"2024-10-24T06:23:43.978915Z","shell.execute_reply":"2024-10-24T06:24:38.398036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ct = 0\nfor x,y in train_data:\n    print(x.shape)\n    print(y.shape)\n    print(x.mean())\n    print(y.sum(0))\n    if ct  > 10:\n        break\n    ct+=1","metadata":{"execution":{"iopub.status.busy":"2024-10-24T06:24:38.401273Z","iopub.execute_input":"2024-10-24T06:24:38.401602Z","iopub.status.idle":"2024-10-24T06:24:51.229451Z","shell.execute_reply.started":"2024-10-24T06:24:38.401568Z","shell.execute_reply":"2024-10-24T06:24:51.228425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 以下にmodelを定義してみよう。\n\n※そのまえに、必ず上のすべてのプログラムをまず実行させてみてね？\n\nmodel =","metadata":{}},{"cell_type":"code","source":"import gc\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T06:24:51.230745Z","iopub.execute_input":"2024-10-24T06:24:51.231064Z","iopub.status.idle":"2024-10-24T06:24:51.378923Z","shell.execute_reply.started":"2024-10-24T06:24:51.231032Z","shell.execute_reply":"2024-10-24T06:24:51.377908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dropout= 0.3\nmodel =nn.Sequential(\n    nn.Conv2d(3,64,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(64),\n    nn.Conv2d(64,64,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(64),\n    nn.MaxPool2d(2,2),\n    nn.Dropout(dropout),\n    nn.Conv2d(64,128,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(128),\n    nn.Conv2d(128,128,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(128),\n    nn.MaxPool2d(2,2),\n    nn.Dropout(dropout),\n    nn.Conv2d(128,256,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(256),\n    nn.Conv2d(256,256,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(256),\n    nn.MaxPool2d(3,3),\n    nn.Dropout(dropout),\n    nn.Conv2d(256,512,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(512),\n    nn.Conv2d(512,512,3,padding='same'),\n    nn.ReLU(),\n    nn.Conv2d(512,512,3,padding='same'),\n    nn.ReLU(),\n    nn.BatchNorm2d(512),\n    nn.MaxPool2d(3,3),\n    nn.Dropout(dropout),\n    nn.Flatten(1),\n    nn.Linear(3072,4096),\n    nn.ReLU(),\n    nn.BatchNorm1d(4096),\n    nn.Linear(4096,4096),\n    nn.ReLU(),\n    nn.BatchNorm1d(4096),\n    nn.Linear(4096,2),    \n    nn.Softmax(-1)\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T06:24:51.380174Z","iopub.execute_input":"2024-10-24T06:24:51.380521Z","iopub.status.idle":"2024-10-24T06:24:51.692194Z","shell.execute_reply.started":"2024-10-24T06:24:51.380471Z","shell.execute_reply":"2024-10-24T06:24:51.691164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_epochs = 20\ntrain(model,train_data,val_data,number_of_epochs,lr=0.001,lr_scheduler=None)\nnumber_of_epochs = 20\ntrain(model,train_data,val_data,number_of_epochs,lr=0.0001,lr_scheduler=None)\n# test(model,test_data)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T06:24:51.693644Z","iopub.execute_input":"2024-10-24T06:24:51.694058Z","iopub.status.idle":"2024-10-24T07:07:12.348876Z","shell.execute_reply.started":"2024-10-24T06:24:51.694013Z","shell.execute_reply":"2024-10-24T07:07:12.348069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"もしモデルが完成したら、以下のコードを実行してください。すると、submit.csvができる。それを提出すると成績が出ますや","metadata":{}},{"cell_type":"code","source":"# !cp /kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip .\n# !ls\n# _unzip(Path(\".\"),\"test.zip\")\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()\n\ndef _pad_image(image, max_width, max_height):\n    width, height = image.size\n    new_image = Image.new(\"RGB\", (max_width, max_height))\n    new_image.paste(image, ((max_width - width) // 2, (max_height - height) // 2))\n    return new_image\n\nmax_width, max_height = (\n            1200,\n            1000,\n        )  # Example di\n\ndef _load_image(path: Path):\n    img = Image.open(path).convert(\"RGB\")\n    img = _pad_image(img, max_width, max_height)\n    img = img.resize((120, 100))\n    # img = transforms.ToTensor()(img)  # Convert PIL Image to Tensor\n    img = np.array(img).astype(np.float32)\n    img = img / 255.0  # Normalize pixel values to [0, 1\n    # 次元をchannelfirstにしたい\n    img = np.transpose(img, (2, 0, 1))  # Change dimensions to (C, H, W)\n\n    return torch.tensor(img,dtype=torch.float32,device='cuda').unsqueeze(0)\n\n\ntest_dir = \"/kaggle/input/notebookc785dff359/dogs-vs-cats-redux-kernels-edition/test/test_pickle/\"\nresults = []\ncount = 0\nimgs = []\nids = []\nlength = len(os.listdir(test_dir))\nwith torch.no_grad():\n    for file in os.listdir(test_dir):\n#         img = _load_image(test_dir+file)\n#         imgs.append(img)\n# #         predict = model(img)[0][0]\n        with open(test_dir+file, \"rb\") as f:\n            img, label = pickle.load(f)\n        imgs.append(torch.tensor(img.astype(np.float32)).to(device).unsqueeze(0))\n        id = int(file.split(\".\")[0])\n        ids.append(id)\n\n        if count%512 ==511 or count == length-1:\n            imgs = torch.cat(imgs,dim=0)\n            predicts = model(imgs)\n            for id,predict in zip(ids,predicts):\n                results.append([int(id),predict])\n            imgs = []\n            ids = []\n            \n            \n        count+=1\n\n    \nresults = [[int(result[0]),float(result[1][1])] for result in results]\n\nimport pandas as pd\nimport numpy as np\ndf = pd.DataFrame(results,columns=['id','label'])\n\ndf.to_csv(\"submit.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T07:07:12.350052Z","iopub.execute_input":"2024-10-24T07:07:12.350349Z","iopub.status.idle":"2024-10-24T07:08:10.560232Z","shell.execute_reply.started":"2024-10-24T07:07:12.350318Z","shell.execute_reply":"2024-10-24T07:08:10.559156Z"},"trusted":true},"execution_count":null,"outputs":[]}]}