{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 画像分類コンペ\n## 目次\n1. ライブラリのインポート\n2. データの整理\n3. 前処理の定義\n4. データセットの作成\n5. モデルの定義\n6. 訓練手法の定義\n7. 訓練を実行\n8. テスト\n9. 提出csvの作成","metadata":{}},{"cell_type":"markdown","source":"## 初期設定","metadata":{}},{"cell_type":"code","source":"import glob\nimport os.path\nimport random\nimport numpy as np\nimport json\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport zipfile\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision\nfrom torchvision import models,transforms\nimport torch.nn.functional as F\n\nfrom sklearn.model_selection import train_test_split\n\nDEVICE =\"cuda\"\nBATCH_SIZE=128","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:22:47.208809Z","iopub.execute_input":"2024-11-13T12:22:47.209360Z","iopub.status.idle":"2024-11-13T12:22:53.490177Z","shell.execute_reply.started":"2024-11-13T12:22:47.209301Z","shell.execute_reply":"2024-11-13T12:22:53.488743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データの整理","metadata":{}},{"cell_type":"code","source":"os.listdir('../input/dogs-vs-cats-redux-kernels-edition')\nos.makedirs('../data', exist_ok=True)\nbase_dir = '../input/dogs-vs-cats-redux-kernels-edition'\ntrain_dir = '../data/train'\ntest_dir = '../data/test'\nwith zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:\n    train_zip.extractall('../data')\n    \nwith zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:\n    test_zip.extractall('../data')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:22:53.492782Z","iopub.execute_input":"2024-11-13T12:22:53.493400Z","iopub.status.idle":"2024-11-13T12:23:15.928267Z","shell.execute_reply.started":"2024-11-13T12:22:53.493352Z","shell.execute_reply":"2024-11-13T12:23:15.926916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\n\ntrain_list, val_list = train_test_split(train_list, test_size=0.1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:23:15.929732Z","iopub.execute_input":"2024-11-13T12:23:15.930144Z","iopub.status.idle":"2024-11-13T12:23:16.108984Z","shell.execute_reply.started":"2024-11-13T12:23:15.930101Z","shell.execute_reply":"2024-11-13T12:23:16.107656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_list[:10])\nprint(test_list[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:23:16.111895Z","iopub.execute_input":"2024-11-13T12:23:16.112341Z","iopub.status.idle":"2024-11-13T12:23:16.119120Z","shell.execute_reply.started":"2024-11-13T12:23:16.112299Z","shell.execute_reply":"2024-11-13T12:23:16.117724Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 前処理の定義","metadata":{}},{"cell_type":"code","source":"class ImageTransform():\n    def __init__(self):\n        self.data_transform = {\n            'train': transforms.Compose([\n                # transforms.Resize((224,224)),\n                transforms.RandomResizedCrop(224),\n                transforms.RandomHorizontalFlip(),\n                transforms.ToTensor(),\n                transforms.Normalize((0.485,0.456,0.406),(0.229,0.224,0.225))\n            ]),\n            'test': transforms.Compose([\n                transforms.Resize(250),\n                transforms.CenterCrop(224),\n                transforms.ToTensor(),\n                transforms.Normalize((0.485,0.456,0.406),(0.229,0.224,0.225))\n            ])\n        }\n    def __call__(self, img, phase):\n        return self.data_transform[phase](img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:23:16.120621Z","iopub.execute_input":"2024-11-13T12:23:16.120997Z","iopub.status.idle":"2024-11-13T12:23:16.131970Z","shell.execute_reply.started":"2024-11-13T12:23:16.120956Z","shell.execute_reply":"2024-11-13T12:23:16.130321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## データセットの作成","metadata":{}},{"cell_type":"code","source":"class MyDataset(data.Dataset):\n    def __init__(self,file_path_list,transform,phase):\n        self.file_list = file_path_list\n        self.transform = transform\n        self.phase = phase\n        \n    def __len__(self):\n        return len(self.file_list)\n    \n    def __getitem__(self,idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img,self.phase)\n        \n        label = img_path.split('/')[-1].split('.')[0]\n        if label == 'dog':\n            label=1\n        elif label == 'cat':\n            label=0\n        else:\n            # これは、phase==testの場合、この場合そのままidを渡すことになる\n            pass\n            \n        return img_transformed,label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:23:16.133743Z","iopub.execute_input":"2024-11-13T12:23:16.134316Z","iopub.status.idle":"2024-11-13T12:23:16.152016Z","shell.execute_reply.started":"2024-11-13T12:23:16.134256Z","shell.execute_reply":"2024-11-13T12:23:16.150527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = ImageTransform()\n\ntrain_dataset = MyDataset(train_list, transform=transform, phase='train')\ntrain_dataloader = data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, pin_memory=True)\n\nval_dataset = MyDataset(val_list, transform=transform, phase='test')\nval_dataloader = data.DataLoader(val_dataset,batch_size=BATCH_SIZE, shuffle = True, num_workers=4, pin_memory=True)\n\ntest_dataset = MyDataset(test_list,transform=transform,phase='test')\ntest_dataloader = data.DataLoader(test_dataset,batch_size=BATCH_SIZE, shuffle = True, num_workers=4, pin_memory=True)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:23:16.153700Z","iopub.execute_input":"2024-11-13T12:23:16.154248Z","iopub.status.idle":"2024-11-13T12:23:16.167838Z","shell.execute_reply.started":"2024-11-13T12:23:16.154162Z","shell.execute_reply":"2024-11-13T12:23:16.166454Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## モデルの定義","metadata":{}},{"cell_type":"code","source":"from torchvision.models import alexnet\n\nmodel =alexnet()\nmodel.train()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:23:16.169344Z","iopub.execute_input":"2024-11-13T12:23:16.169842Z","iopub.status.idle":"2024-11-13T12:23:16.938181Z","shell.execute_reply.started":"2024-11-13T12:23:16.169798Z","shell.execute_reply":"2024-11-13T12:23:16.936637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.classifier[-1]= nn.Linear(4096,2)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 訓練手法の定義\n","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n\n# 分類問題用のtrain_model\ndef train_model(model, train_loader, val_loader, optimizer, criterion, epochs):\n    def _train(epoch):\n        epoch_loss = 0\n        epoch_accuracy = 0\n        # tqdmで進捗バーを表示し、postfixで訓練の途中結果を表示\n        with tqdm(train_loader, desc=\"Training\", leave=True) as pbar:\n            for data, label in pbar:\n                data = data.to(DEVICE)\n                label = label.to(DEVICE)\n                \n                output = model(data)\n                loss = criterion(output, label)\n                \n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                \n                acc = (output.argmax(dim=1) == label).float().mean()\n                epoch_accuracy += acc / len(train_loader)\n                epoch_loss += loss / len(train_loader)\n                \n                # postfixでtrain_loss, acc, learning rateを表示\n                pbar.set_postfix({\n                    'Train Loss': epoch_loss.item(),\n                    'Train Accuracy': epoch_accuracy.item(),\n                    'Learning Rate': optimizer.param_groups[0]['lr']\n                })\n        \n        print(f\"Epoch: {epoch+1}, Train Accuracy: {epoch_accuracy:.4f}, Train Loss: {epoch_loss:.4f}\")\n    def _val(epoch):\n        # 検証の進行状況を表示\n        epoch_val_accuracy = 0\n        epoch_val_loss = 0\n        with torch.no_grad():\n            with tqdm(val_loader, desc=\"Validation\", leave=True) as pbar:\n                for data, label in pbar:\n                    data = data.to(DEVICE)\n                    label = label.to(DEVICE)\n                    \n                    val_output = model(data)\n                    val_loss = criterion(val_output, label)\n                    \n                    acc = (val_output.argmax(dim=1) == label).float().mean()\n                    epoch_val_accuracy += acc / len(val_loader)\n                    epoch_val_loss += val_loss / len(val_loader)\n                    \n                    # postfixでval_loss, val_accを表示\n                    pbar.set_postfix({\n                        'Val Loss': epoch_val_loss.item(),\n                        'Val Accuracy': epoch_val_accuracy.item()\n                    })\n        \n        print(f\"Epoch: {epoch+1}, Validation Accuracy: {epoch_val_accuracy:.4f}, Validation Loss: {epoch_val_loss:.4f}\")\n\n    \n\n    \n    model = model.to(DEVICE)\n\n    _val(-1)\n    \n    for epoch in range(epochs):\n\n        print(f\"Epoch {epoch+1}/{epochs}\")\n        \n        _train(epoch)\n        \n        _val(epoch)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:37:52.274528Z","iopub.execute_input":"2024-11-13T11:37:52.275431Z","iopub.status.idle":"2024-11-13T11:37:52.287995Z","shell.execute_reply.started":"2024-11-13T11:37:52.27539Z","shell.execute_reply":"2024-11-13T11:37:52.28705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer= optim.Adam(params = model.parameters(), lr=0.0005)\ncriterion = nn.CrossEntropyLoss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:41:15.237366Z","iopub.execute_input":"2024-11-13T11:41:15.237897Z","iopub.status.idle":"2024-11-13T11:41:15.243623Z","shell.execute_reply.started":"2024-11-13T11:41:15.237858Z","shell.execute_reply":"2024-11-13T11:41:15.24268Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 訓練の実行","metadata":{}},{"cell_type":"code","source":"train_model(model,train_dataloader,val_dataloader,optimizer,criterion,10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T11:41:18.010639Z","iopub.execute_input":"2024-11-13T11:41:18.011062Z","iopub.status.idle":"2024-11-13T11:58:29.150712Z","shell.execute_reply.started":"2024-11-13T11:41:18.011015Z","shell.execute_reply":"2024-11-13T11:58:29.148957Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 提出csvの作成","metadata":{}},{"cell_type":"code","source":"dog_probs = []\nmodel.eval()\nwith torch.no_grad():\n    for data, fileid in test_dataloader:\n        data = data.to(DEVICE)\n        preds = model(data)\n        preds_list = F.softmax(preds, dim=1)[:, 1].tolist()\n        dog_probs += list(zip(list(fileid), preds_list))\n\ndog_probs.sort(key = lambda x : int(x[0]))\ndog_probs[:10]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:04:18.437957Z","iopub.execute_input":"2024-11-13T12:04:18.438349Z","iopub.status.idle":"2024-11-13T12:04:44.736231Z","shell.execute_reply.started":"2024-11-13T12:04:18.438305Z","shell.execute_reply":"2024-11-13T12:04:44.735202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"idx = list(map(lambda x: x[0],dog_probs))\nprob = list(map(lambda x: x[1],dog_probs))\n\nsubmission = pd.DataFrame({'id':idx,'label':prob})\nsubmission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:07:23.136302Z","iopub.execute_input":"2024-11-13T12:07:23.136924Z","iopub.status.idle":"2024-11-13T12:07:23.194022Z","shell.execute_reply.started":"2024-11-13T12:07:23.136885Z","shell.execute_reply":"2024-11-13T12:07:23.193044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('result.csv',index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:07:24.274536Z","iopub.execute_input":"2024-11-13T12:07:24.274993Z","iopub.status.idle":"2024-11-13T12:07:24.326198Z","shell.execute_reply.started":"2024-11-13T12:07:24.274934Z","shell.execute_reply":"2024-11-13T12:07:24.325442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## テスト","metadata":{}},{"cell_type":"code","source":"import random\n\nid_list = []\nclass_ = {0: 'cat', 1: 'dog'}\n\nfig, axes = plt.subplots(2, 5, figsize=(20, 12), facecolor='w')\n\nfor ax in axes.ravel():\n    \n    i = random.choice(submission['id'].values)\n    \n    label = submission.loc[submission['id'] == i, 'label'].values[0]\n    if label > 0.5:\n        label = 1\n    else:\n        label = 0\n        \n    img_path = os.path.join(test_dir, '{}.jpg'.format(i))\n    img = Image.open(img_path)\n    \n    ax.set_title(class_[label])\n    ax.imshow(img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T12:07:25.377252Z","iopub.execute_input":"2024-11-13T12:07:25.378149Z","iopub.status.idle":"2024-11-13T12:07:27.8721Z","shell.execute_reply.started":"2024-11-13T12:07:25.378108Z","shell.execute_reply":"2024-11-13T12:07:27.871135Z"}},"outputs":[],"execution_count":null}]}