{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\n\nwith zipfile.ZipFile('../input/dogs-vs-cats-redux-kernels-edition/test.zip') as existing_zip:\n    existing_zip.extractall()\nwith zipfile.ZipFile('../input/dogs-vs-cats-redux-kernels-edition/train.zip') as existing_zip:\n    existing_zip.extractall()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input/dogs-vs-cats-redux-kernels-edition","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mkdir cat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mkdir dog","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nimport glob\nimport os\n\ndef move_glob(dst_path, pathname, recursive=True):\n    for p in glob.glob(pathname, recursive=recursive):\n        shutil.move(p, dst_path)\n\nmove_glob('./dog', 'dog*.jpg')\n\nmove_glob('./cat', 'cat*.jpg')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install openpyxl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ../","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:40.551607Z","iopub.execute_input":"2021-07-30T08:39:40.552064Z","iopub.status.idle":"2021-07-30T08:39:40.558647Z","shell.execute_reply.started":"2021-07-30T08:39:40.552012Z","shell.execute_reply":"2021-07-30T08:39:40.557473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch # 基本モジュール\nfrom torch.autograd import Variable # 自動微分用\nimport torch.nn as nn # ネットワーク構築用\nimport torch.optim as optim # 最適化関数\nimport torch.nn.functional as F # ネットワーク用の様々な関数\nimport torch.utils.data # データセット読み込み関連\nfrom torchvision import transforms\nfrom torch.utils.tensorboard import SummaryWriter\nimport torchvision.models as models\nimport math\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\nimport matplotlib.pyplot as plt\nimport cloudpickle\nimport random\nimport cv2\n\nimport pandas as pd\nfrom PIL import Image,ImageFilter\nimport numpy as np\nimport openpyxl as px\n\nimport sys\nimport time\nimport datetime\nLABEL_IDX = 1\nIMG_IDX = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:40:36.21881Z","iopub.execute_input":"2021-07-30T08:40:36.219326Z","iopub.status.idle":"2021-07-30T08:40:36.250868Z","shell.execute_reply.started":"2021-07-30T08:40:36.21929Z","shell.execute_reply":"2021-07-30T08:40:36.249047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('./test.txt', mode='w') as f:\n  f.write(\"name,label\")\n  for num in range(1,12501):\n    f.write('\\n')\n    f.write(\"test/\" + (str)(num) + \",0\")\n    \nwith open('./dog.txt', mode='w') as f:\n  f.write(\"name,label\")\n  for num in range(1,12500):\n    f.write('\\n')\n    f.write(\"dog/dog.\" + (str)(num) + \",0\")\n    \nwith open('./cat.txt', mode='w') as f:\n  f.write(\"name,label\")\n  for num in range(1,12500):\n    f.write('\\n')\n    f.write(\"cat/cat.\" + (str)(num) + \",1\")\n    \n# 学習データのパス\nTRAIN_PATH = [\n    r\"./cat.txt\",\n    r\"./dog.txt\",\n    r\"./test.txt\"\n]\n\n# ミニバッチ\nBATCH_SIZE = 100","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.861197Z","iopub.status.idle":"2021-07-30T08:39:42.861667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.865401Z","iopub.status.idle":"2021-07-30T08:39:42.866143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#データロード及びリサイズ\nclass uDataset(Dataset):\n\n    def __init__(self, csv_file_path, root_dir, training=True):\n        #pandasでcsvデータの読み出し\n        self.image_dataframe = pd.read_csv(csv_file_path)\n        self.root_dir = root_dir\n\n        #学習or推論\n        self.training = training\n\n    def __len__(self):\n        return len(self.image_dataframe)\n\n    def __getitem__(self, idx):\n        #dataframeから画像へのパスとラベルを読み出す\n        label = self.image_dataframe.iat[idx, LABEL_IDX]\n\n        img_name = os.path.join(self.root_dir, self.image_dataframe.iat[idx, IMG_IDX])\n\n        path = img_name + '.jpg'\n        val = np.mean(Image.open(path))\n\n        # 選択したインデックスの画像の読み込み\n        image = Image.open(path)\n\n        # 画像への前処理\n        if image.width != 256 or image.height != 256:\n            image = image.resize((256, 256),Image.BICUBIC)\n            \n        \n        # numpy型への変換\n        image = np.concatenate([image],2)\n        imageNP = np.array(image).astype(np.float32)\n\n\n        # 標準化を行う\n        ave_1 = np.mean(imageNP[:,:,0])\n        ave_2 = np.mean(imageNP[:,:,1])\n        ave_3 = np.mean(imageNP[:,:,2])\n        std_1 = np.std(imageNP[:,:,0])\n        std_2 = np.std(imageNP[:,:,1])\n        std_3 = np.std(imageNP[:,:,2])\n        \n        \n        if std_1 > 0:\n            imageNP[:,:,0] = (imageNP[:,:,0] - ave_1) / std_1\n        if std_2 > 0:\n            imageNP[:,:,1] = (imageNP[:,:,1] - ave_2) / std_2\n        if std_3 > 0:\n            imageNP[:,:,2] = (imageNP[:,:,2] - ave_3) / std_3\n\n        image = transforms.ToTensor()(imageNP)\n\n\n        return image, label\n    \n    def filepath(self, idx):\n        img_name = os.path.join(self.root_dir, self.image_dataframe.iat[idx, IMG_IDX])\n        return img_name","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.868074Z","iopub.status.idle":"2021-07-30T08:39:42.868801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#spp\ndef spatial_pyramid_pool(previous_conv, num_sample, previous_conv_size, out_pool_size):\n\n    #print(previous_conv.size())\n    for i in range(len(out_pool_size)):\n        h_wid = int(math.ceil(previous_conv_size[0] / out_pool_size[i]))\n        w_wid = int(math.ceil(previous_conv_size[1] / out_pool_size[i]))\n        h_pad = (h_wid*out_pool_size[i] - previous_conv_size[0] + 1)/2\n        w_pad = (w_wid*out_pool_size[i] - previous_conv_size[1] + 1)/2\n        maxpool = nn.MaxPool2d((h_wid, w_wid), stride=(h_wid, w_wid), padding=(int(h_pad), int(w_pad)))\n        x = maxpool(previous_conv)\n        if(i == 0):\n            spp = x.view(num_sample,-1)\n            #print(\"spp size:\",spp.size())\n        else:\n            #print(\"size:\",spp.size())\n            spp = torch.cat((spp,x.view(num_sample,-1)), 1)\n    return spp\n\nclass sppnet(nn.Module):\n    def __init__(self, n_class=2):\n        super(sppnet, self).__init__()\n        \n        self.inputLayer = nn.Sequential(\n            nn.Conv2d(3, 32, kernel_size=1, stride=1, bias=True),\n            nn.BatchNorm2d(32),\n            #nn.Dropout(p = 0.3),\n            nn.ELU(inplace=True),\n            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(64),\n            #nn.Dropout(p = 0.3),\n            nn.ELU(inplace=True),\n            nn.MaxPool2d(2, stride=2)\n        )\n\n        self.layer1 = nn.Sequential(\n            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(128),\n            #nn.Dropout(p = 0.5),\n            nn.ELU(inplace=True),\n            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),\n            nn.ELU(inplace=True),\n            nn.MaxPool2d(2, stride=2)\n        )\n        self.layer2 = nn.Sequential(\n            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(256),\n            #nn.Dropout(p = 0.5),\n            nn.ELU(inplace=True),\n            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),\n            nn.ELU(inplace=True),\n            #nn.MaxPool2d(2, stride=2)\n        )\n        self.fc1 = nn.Linear(35*256, 1024)\n        self.fc2 = nn.Linear(1024, n_class)\n    \n    def forward(self, x):\n        #print(x.shape)\n        x = self.inputLayer(x)\n        #print(x.shape)\n        x = self.layer1(x)\n        #print(x.shape)\n        x = self.layer2(x)\n        #print(x.shape)\n        x = spatial_pyramid_pool(x,int(x.size(0)),[int(x.size(2)),int(x.size(3))],[1,3,5])\n        x = F.dropout(self.fc1(x), p=0.8, training=self.training)\n        x = self.fc2(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.870295Z","iopub.status.idle":"2021-07-30T08:39:42.870953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loadData():\n    # データセットの作成\n    dataset1 = uDataset(TRAIN_PATH[0], r\"./train/\")\n    dataset2 = uDataset(TRAIN_PATH[1], r\"./train/\")\n    dataset3 = uDataset(TRAIN_PATH[2], r\"./\")\n\n    trainDataset = dataset1 + dataset2\n    testDataset = dataset3\n\n    # データの読み込み\n    trainDataloader = torch.utils.data.DataLoader(trainDataset, batch_size=BATCH_SIZE, shuffle=True)\n    testDataloader = torch.utils.data.DataLoader(testDataset, batch_size=256 , shuffle=True)\n\n    return trainDataloader, testDataloader\n\n# ハイパーパラメータの設定\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")                   # 学習に使用するデバイス\nmodel = sppnet().to(device)\ncriterion = nn.CrossEntropyLoss()                                                       # loss関数\noptimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-08)\ntrainDataloader, testDataloader = loadData()                                            # データの読み込み\n\n    \ndef train(dataLoader, epoch):\n    model.train()\n    trainLoss = 0\n    correct = 0\n\n    for batchIdx, (image, label) in enumerate(dataLoader):\n        image, label = image.to(device), label.to(device)\n        optimizer.zero_grad()\n        output = model(image)\n        loss = criterion(output, label)\n        trainLoss += loss.item()\n        pred = output.argmax(dim=1, keepdim=True)\n        correct += pred.eq(label.view_as(pred)).sum().item()\n            \n        loss.backward()\n        optimizer.step()\n\n        if batchIdx * BATCH_SIZE % 16 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)], Loss: {:.6f}'.format(\n                epoch, batchIdx * BATCH_SIZE, len(dataLoader.dataset),\n                100. * batchIdx / len(dataLoader), loss.item()))\n        \n    return trainLoss, correct\n\ndef test(dataLoader):\n    model.eval()\n    testLoss = 0\n    correct = 0\n    with torch.no_grad():\n        for(image, label) in dataLoader:\n            \n            #print()\n            image, label = image.to(device), label.to(device)\n            output = model(image)\n            loss = criterion(output, label)\n            testLoss += loss.item()\n            pred = output.argmax(dim=1, keepdim=True)\n            correct += pred.eq(label.view_as(pred)).sum().item()\n\n        testLoss /= len(dataLoader)\n        print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n            testLoss, correct, len(dataLoader.dataset),\n            100. * correct / len(dataLoader.dataset)))\n\n    return testLoss, correct\n\ndef learning(model):\n\n    #学習\n    numEpochs = 50\n\n    lossList = []\n    accList = []\n\n    # 学習の収束結果をtensorboardに出力\n    writer = SummaryWriter(log_dir=\"./logs/sgd\")\n    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n\n    for epoch in range(1, numEpochs + 1):\n        loss, acc = train(trainDataloader, epoch)\n\n        #test(testDataloader)\n\n        writer.add_scalar(\"loss/train\", loss / len(trainDataloader), epoch)\n        writer.add_scalar(\"accuracy/train\", 100. * float(acc) / len(trainDataloader.dataset), epoch)\n\n        lossList.append(loss / len(trainDataloader))\n        accList.append(100. * float(acc) / len(trainDataloader.dataset))\n\n        # 10エポック毎に結果を保存\n        if epoch % 100 == 0:\n            savePath = \"./cnn_interlude_\" + \"{:05}\".format(epoch)\n            torch.save(model.state_dict(), savePath + \".weight\")\n            with open(savePath + \".pkl\", 'wb') as f:\n                cloudpickle.dump(model, f)\n        \n        scheduler.step()\n\n    writer.close()\n    \n    # 最終結果を保存\n    torch.save(model.state_dict(), './cnn_result.pkl')\n    with open('cnn_result.pkl', 'wb') as f:\n        cloudpickle.dump(model, f)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-07-30T08:39:42.873012Z","iopub.status.idle":"2021-07-30T08:39:42.873729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.875147Z","iopub.status.idle":"2021-07-30T08:39:42.875831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"       \ndef inference(folderIndex):\n\n    \n    book = px.Workbook()\n    book.save('../' + 'submission.xlsx')\n    wb = px.Workbook()\n    ws = wb.active\n    #Sheetの名前を設定\n    ws.title = \"Sheet_1\"\n\n\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    print(device)\n    \n\n    # データセットの作成\n    dataset = uDataset(TRAIN_PATH[folderIndex], r\"./\", training=False)\n    dataLoader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n\n    with open('./cnn_result.pkl', 'rb') as f:\n        model = cloudpickle.load(f)\n        #shutil.copy2('./cnn_result.pkl', saveFolderPath + \"cnn_result.pkl\")\n        #print(model)\n        model.to(device)\n\n        model.eval()\n        a = 2\n        ws.cell(row = 1, column = 2, value = \"label\")\n        ws.cell(row = 1, column = 1, value = \"id\")\n\n        with torch.no_grad():\n            for idx, (image, label) in enumerate(dataLoader):\n                image, label = image.to(device), label.to(device)\n                start2 = time.time()\n                output = model(image)\n                #print(model(image))\n                \n                # softmaxで犬とする確率の確率を算出\n                outputS = F.softmax(model(image), dim=1)\n                \n                #csvファイルに書き込み\n                ws.cell(row = a, column = 2, value = float(outputS[0][0]))\n                ws.cell(row = a, column = 1, value = a-1)\n                a += 1\n                print(a)\n                \n    wb.save('./' + 'submission.xlsx')","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.877643Z","iopub.status.idle":"2021-07-30T08:39:42.878339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    # 学習モデルのサイズ\n    print('Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0))\n    learning(model)\n    idx = 2\n    inference(idx)\n    change_csv = pd.read_excel('./' + 'submission.xlsx', sheet_name=0)\n    change_csv.to_csv('./' + 'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.879838Z","iopub.status.idle":"2021-07-30T08:39:42.880528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-07-30T08:39:42.881818Z","iopub.status.idle":"2021-07-30T08:39:42.8825Z"},"trusted":true},"execution_count":null,"outputs":[]}]}