{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport os\nimport pandas as pd\nimport skimage as sk\nfrom skimage import io as skio, transform\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nimport time\nimport copy\nimport random\nimport torchvision.models as models\nfrom PIL import Image # to read images\nfrom PIL import Image, ImageOps\nfrom torchvision.transforms import ToTensor, ToPILImage\nimport json\nimport time\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import KFold\nimport cv2\nimport torch.nn.functional as F\nfrom mpl_toolkits.axes_grid1 import ImageGrid","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:06:33.995543Z","iopub.execute_input":"2021-11-13T02:06:33.995794Z","iopub.status.idle":"2021-11-13T02:06:34.003682Z","shell.execute_reply.started":"2021-11-13T02:06:33.995765Z","shell.execute_reply":"2021-11-13T02:06:34.002951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_face_and_points(data_folder):\n    point_file_list = []\n    face_file_list = []\n    face_list = []\n    point_list = []\n    for f in os.listdir(data_folder):\n        if f[-3:] == 'asf':\n            point_file_list.append(f)\n        if f[-3:] == 'jpg':\n            face_file_list.append(f) \n        \n    point_file_list = sorted(point_file_list)\n    face_file_list = sorted(face_file_list)\n    \n    for i in range(len(face_file_list)):\n        img = skio.imread(data_folder + '/' + face_file_list[i])\n        img = sk.img_as_float(img)\n        face_list.append(img)\n        \n        point = pd.read_csv(data_folder + '/' + point_file_list[i])\n        point = point.iloc[13:71,].reset_index(drop=True)\n        point['x_cord'] = point.iloc[:,0].apply(lambda x: float(x.split()[2]))\n        point['y_cord'] = point.iloc[:,0].apply(lambda x: float(x.split()[3]))\n        \n        point = point[['x_cord','y_cord']]\n        \n        width = 640\n        height = 480\n        point['x_cord'] = point['x_cord']*width\n        point['y_cord'] = point['y_cord']*height\n        \n        \n        point = point.to_numpy()\n        point_list.append(point)\n        \n    return np.array(face_list), np.array(point_list), np.array(face_file_list)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:05.510770Z","iopub.execute_input":"2021-11-13T01:21:05.511295Z","iopub.status.idle":"2021-11-13T01:21:05.523424Z","shell.execute_reply.started":"2021-11-13T01:21:05.511225Z","shell.execute_reply":"2021-11-13T01:21:05.522769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '../input/face-dataset/imm_face_db'","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:05.524799Z","iopub.execute_input":"2021-11-13T01:21:05.525220Z","iopub.status.idle":"2021-11-13T01:21:05.534392Z","shell.execute_reply.started":"2021-11-13T01:21:05.525179Z","shell.execute_reply":"2021-11-13T01:21:05.533635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"face_list, point_list, face_file_list = save_face_and_points(DATA_DIR)\n\nlst = []\nfor f in face_file_list:\n    a = DATA_DIR+'/'+f\n    lst.append(a)\npic_path = np.array(lst)\n\ncolumn_names = ['pic_path', 'x', 'y']\ndata = pd.DataFrame(columns = column_names)\n\ndata['pic_path'] = pic_path\ndata['x'] = list(point_list[:,:,0])\ndata['y'] = list(point_list[:,:,1])","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:05.535573Z","iopub.execute_input":"2021-11-13T01:21:05.535948Z","iopub.status.idle":"2021-11-13T01:21:09.244964Z","shell.execute_reply.started":"2021-11-13T01:21:05.535906Z","shell.execute_reply":"2021-11-13T01:21:09.244229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    print('> SEEDING DONE')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.279058Z","iopub.execute_input":"2021-11-13T01:21:09.279521Z","iopub.status.idle":"2021-11-13T01:21:09.287094Z","shell.execute_reply.started":"2021-11-13T01:21:09.279480Z","shell.execute_reply":"2021-11-13T01:21:09.286130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \ndef get_logger(filename='log'):\n    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nlogger = get_logger()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.288679Z","iopub.execute_input":"2021-11-13T01:21:09.289216Z","iopub.status.idle":"2021-11-13T01:21:09.299813Z","shell.execute_reply.started":"2021-11-13T01:21:09.289176Z","shell.execute_reply":"2021-11-13T01:21:09.299034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.301424Z","iopub.execute_input":"2021-11-13T01:21:09.302082Z","iopub.status.idle":"2021-11-13T01:21:09.308009Z","shell.execute_reply.started":"2021-11-13T01:21:09.302044Z","shell.execute_reply":"2021-11-13T01:21:09.307219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rotate_augmentation(image, x, y, angle):\n    h, w = image.shape[:2]\n    M = cv2.getRotationMatrix2D((w//2,h//2), angle, 1.0)\n    angle_rad = -angle*np.pi/180\n    rotated_image = cv2.warpAffine(image, M, (w,h), flags=cv2.INTER_CUBIC)\n    x = x-w//2\n    y = y-h//2\n    rotated_x = x*np.cos(angle_rad)-y*np.sin(angle_rad)+w//2\n    rotated_y = x*np.sin(angle_rad)+y*np.cos(angle_rad)+h//2\n    \n    return rotated_image, rotated_x, rotated_y","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.309519Z","iopub.execute_input":"2021-11-13T01:21:09.310348Z","iopub.status.idle":"2021-11-13T01:21:09.318144Z","shell.execute_reply.started":"2021-11-13T01:21:09.310309Z","shell.execute_reply":"2021-11-13T01:21:09.317364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def alter_brightness(image, x, y):\n    if random.random() < 0.5:\n        altered_image = np.clip(image*(random.randrange(50,100,1)/100), 0.0, 1.0)  \n    else:\n        altered_image = np.clip(image*(random.randrange(100,200,1)/100), 0.0, 1.0)    \n    return altered_image, x, y","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.319512Z","iopub.execute_input":"2021-11-13T01:21:09.320029Z","iopub.status.idle":"2021-11-13T01:21:09.326962Z","shell.execute_reply.started":"2021-11-13T01:21:09.319991Z","shell.execute_reply":"2021-11-13T01:21:09.326264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.329239Z","iopub.execute_input":"2021-11-13T01:21:09.330652Z","iopub.status.idle":"2021-11-13T01:21:09.359926Z","shell.execute_reply.started":"2021-11-13T01:21:09.330621Z","shell.execute_reply":"2021-11-13T01:21:09.359020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self,data, transform=True):\n        self.data = data\n        self.transform = transform\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, i):\n        row = self.data.iloc[i]\n        ori_image = Image.open(row.pic_path)\n        ori_image = ImageOps.grayscale(ori_image)\n        ori_image = ToTensor()(ori_image).squeeze()\n        x = row.x\n        y = row.y\n        landmarks = np.column_stack((row.x,row.y))\n        im = ori_image\n        h, w = im.shape[:2]\n            #resize\n        im = torch.unsqueeze(im, dim=0)\n        im = transforms.Resize((180,240))(im).squeeze().numpy()\n        #scale points\n        landmarks = landmarks * [240/w, 180/h]\n        scaled_landmarks = landmarks * [1/240, 1/180]\n        image = im\n        x=landmarks[:,0]\n        y=landmarks[:,1]\n        if self.transform:\n            #aug\n            if random.random() < 0.6:\n                # select random angle between -45 and 45\n                angle = random.randrange(-45, 45, 1) \n                temp_image, temp_x, temp_y = rotate_augmentation(image, x, y, angle)\n                h,w=temp_image.shape\n                if sum(temp_x > w)==0 and sum(temp_x < 0)==0 and sum(temp_y > h)==0 and sum(temp_y < 0)==0:\n                    image, x, y = temp_image, temp_x, temp_y\n            if random.random() < 0.5:\n                image, x, y =  alter_brightness(image, x, y)\n        landmarks = np.column_stack((x,y))\n        landmarks = torch.from_numpy(landmarks)\n        image = torch.from_numpy(image)\n        image = torch.unsqueeze(image, 0)\n            \n        output = {'image': image, 'pts': landmarks, 'scaled_pts': scaled_landmarks.flatten(),'h': h, 'w': w, 'gt_x':x, 'gt_y':y}\n        return output\n#         return im,landmarks,h,w,bbox","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.468730Z","iopub.execute_input":"2021-11-13T01:21:09.469066Z","iopub.status.idle":"2021-11-13T01:21:09.484647Z","shell.execute_reply.started":"2021-11-13T01:21:09.469028Z","shell.execute_reply":"2021-11-13T01:21:09.483913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfull_train_dataset = TrainDataset(train, transform=True)\nfull_train_dataloader = DataLoader(full_train_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.486156Z","iopub.execute_input":"2021-11-13T01:21:09.486364Z","iopub.status.idle":"2021-11-13T01:21:09.495941Z","shell.execute_reply.started":"2021-11-13T01:21:09.486340Z","shell.execute_reply":"2021-11-13T01:21:09.495046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor i_batch, sample_batched in enumerate(full_train_dataloader):\n    plt.figure()\n    plt.imshow(sample_batched['image'][0][0], cmap='gray')\n    plt.scatter(sample_batched['pts'][0][:,0], sample_batched['pts'][0][:,1], s=10, marker='.', c='r')\n    plt.savefig(f'part2_Sample{count}.jpeg')\n    plt.show()\n    count += 1\n    if count > 5:\n        break","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:09.497314Z","iopub.execute_input":"2021-11-13T01:21:09.497680Z","iopub.status.idle":"2021-11-13T01:21:11.223914Z","shell.execute_reply.started":"2021-11-13T01:21:09.497642Z","shell.execute_reply":"2021-11-13T01:21:11.223262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(train_loader, model, optimizer, criterion, device):\n    model = model.train()\n    t = time.time()\n    losses = AverageMeter()\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for i_batch, sample_batched in pbar:\n        im = sample_batched['image']\n        batch_size = im.size(0)\n        im = im.to(device, dtype=torch.float)\n        pts = sample_batched['scaled_pts']\n        pts = pts.to(device, dtype=torch.float)\n        optimizer.zero_grad()\n        output = model(im)\n        loss = criterion(output, pts)\n        loss.backward()\n        optimizer.step()\n        losses.update(loss.item(), batch_size)\n    return losses.avg","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.225077Z","iopub.execute_input":"2021-11-13T01:21:11.225595Z","iopub.status.idle":"2021-11-13T01:21:11.234311Z","shell.execute_reply.started":"2021-11-13T01:21:11.225559Z","shell.execute_reply":"2021-11-13T01:21:11.233543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_fn(vdataloader, model, criterion):\n    model.eval()\n    t = time.time()\n    losses = AverageMeter()\n    pbar = tqdm(enumerate(vdataloader), total=len(vdataloader))\n    for i_batch, sample_batched in pbar:\n        im = sample_batched['image']\n        batch_size = im.size(0)\n        im = im.to(device, dtype=torch.float)\n        pts = sample_batched['scaled_pts']\n        pts = pts.to(device, dtype=torch.float)\n        with torch.no_grad():\n            output = net(im)\n        loss = criterion(output, pts)\n        losses.update(loss.item(), batch_size)\n    return losses.avg","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.235602Z","iopub.execute_input":"2021-11-13T01:21:11.235894Z","iopub.status.idle":"2021-11-13T01:21:11.244345Z","shell.execute_reply.started":"2021-11-13T01:21:11.235845Z","shell.execute_reply":"2021-11-13T01:21:11.243523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = train.copy()\nFold = KFold(n_splits=5, shuffle=True, random_state=42)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds['pic_path'])):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.245578Z","iopub.execute_input":"2021-11-13T01:21:11.245998Z","iopub.status.idle":"2021-11-13T01:21:11.260260Z","shell.execute_reply.started":"2021-11-13T01:21:11.245925Z","shell.execute_reply":"2021-11-13T01:21:11.259545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(fold_num,df=folds):\n    train_df = df[df.fold != fold_num].reset_index(drop=True)\n    valid_df = df[df.fold == fold_num].reset_index(drop=True)\n    \n    train_dataset = TrainDataset(train_df)\n    valid_dataset = TrainDataset(valid_df)\n\n    train_loader = DataLoader(train_dataset, batch_size=8, \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=16, \n                              num_workers=2, shuffle=False, pin_memory=True)\n    print('train size:',train_df.shape[0])\n    print('valid size:',valid_df.shape[0])\n    \n    return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.261477Z","iopub.execute_input":"2021-11-13T01:21:11.262219Z","iopub.status.idle":"2021-11-13T01:21:11.269685Z","shell.execute_reply.started":"2021-11-13T01:21:11.262181Z","shell.execute_reply":"2021-11-13T01:21:11.269003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Part2_Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1,32,7)\n        self.conv2 = nn.Conv2d(32,64,7)\n        self.pool1 = nn.MaxPool2d(2,stride=2)\n        self.conv3 = nn.Conv2d(64,32,5)\n        self.pool2 = nn.MaxPool2d(2,stride=2)\n        self.conv4 = nn.Conv2d(32,16,5)\n        self.pool3 = nn.MaxPool2d(2,stride=2)\n        self.conv5 = nn.Conv2d(16,16,5)\n        self.pool4 = nn.MaxPool2d(2,stride=2)\n        \n        \n        self.fc1 = nn.Linear(1120,200)\n        self.fc2 = nn.Linear(200,58*2)\n        \n        \n\n    def forward(self, x):  \n        x = self.conv1(x)\n        x = F.relu(x)\n        x = self.conv2(x)\n        x = F.relu(x)        \n        x = self.pool1(x)\n        x = self.conv3(x)\n        x = F.relu(x)\n        x = self.pool2(x)\n        x = self.conv4(x)\n        x = F.relu(x)\n        x = self.pool3(x)\n        x = self.conv5(x)\n        x = F.relu(x)\n        x = self.pool4(x)\n        x = torch.flatten(x, 1)\n        x = self.fc1(x)\n        x = F.relu(x)\n        x = self.fc2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.270789Z","iopub.execute_input":"2021-11-13T01:21:11.273720Z","iopub.status.idle":"2021-11-13T01:21:11.286604Z","shell.execute_reply.started":"2021-11-13T01:21:11.273687Z","shell.execute_reply":"2021-11-13T01:21:11.285820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Part2_Model()\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.288181Z","iopub.execute_input":"2021-11-13T01:21:11.288777Z","iopub.status.idle":"2021-11-13T01:21:11.300310Z","shell.execute_reply.started":"2021-11-13T01:21:11.288738Z","shell.execute_reply":"2021-11-13T01:21:11.299577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0\ntrain_loader, valid_loader = prepare_loaders(fold_num=fold)\nseed=42\nset_seed(seed)\nepochs=300\nnet = model\nnet = net.to(device, dtype=torch.float)\n# Define loss_fn\ncriterion = nn.MSELoss()\nlearning_rate = 1e-3\noptimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)\nstart = time.time()\n\nlog_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))\nbest_loss = np.inf\nfor epoch in range(epochs):\n    train_loss = train_fn(train_loader, net, optimizer, criterion, device)\n    valid_loss = valid_fn(valid_loader, net, criterion)\n    log_row = {'EPOCH': epoch, \n               'TRAIN_LOSS': train_loss,\n               'VALID_LOSS': valid_loss,\n              }\n    print('train_loss: ',train_loss)\n    print('valid_loss: ',valid_loss)\n    log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)\n    if valid_loss < best_loss:\n        logger.info(f'epoch{epoch} save best model... {valid_loss}')\n        best_loss = valid_loss\n        torch.save(net.state_dict(), f\"fold{fold}_epoch{epoch}_seed{seed}.pth\")\nend = time.time()\nprint(\"time= \" + str(end-start))","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:21:11.301663Z","iopub.execute_input":"2021-11-13T01:21:11.302119Z","iopub.status.idle":"2021-11-13T01:34:26.951245Z","shell.execute_reply.started":"2021-11-13T01:21:11.302086Z","shell.execute_reply":"2021-11-13T01:34:26.949255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(log_df.EPOCH, log_df.TRAIN_LOSS, label=\"Train\")\nplt.plot(log_df.EPOCH, log_df.VALID_LOSS, label=\"Validation\")\nplt.xlabel('epoch')\nplt.ylabel('loss')\nplt.legend(loc=\"upper right\")\nplt.savefig('part2_loss_graph.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:38:22.201727Z","iopub.execute_input":"2021-11-13T01:38:22.202437Z","iopub.status.idle":"2021-11-13T01:38:22.458600Z","shell.execute_reply.started":"2021-11-13T01:38:22.202401Z","shell.execute_reply":"2021-11-13T01:38:22.457939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(torch.load('./fold0_epoch249_seed42.pth'))\nmodel = model.to(device, dtype=torch.float)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:35:27.548838Z","iopub.execute_input":"2021-11-13T01:35:27.549373Z","iopub.status.idle":"2021-11-13T01:35:27.559529Z","shell.execute_reply.started":"2021-11-13T01:35:27.549333Z","shell.execute_reply":"2021-11-13T01:35:27.558783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:35:28.970807Z","iopub.execute_input":"2021-11-13T01:35:28.971225Z","iopub.status.idle":"2021-11-13T01:35:28.978913Z","shell.execute_reply.started":"2021-11-13T01:35:28.971188Z","shell.execute_reply":"2021-11-13T01:35:28.977869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visual_dataset = TrainDataset(train,transform=False)\n\nvisual_loader = DataLoader(visual_dataset, batch_size=1, \n                          num_workers=2, shuffle=True, pin_memory=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:35:32.611013Z","iopub.execute_input":"2021-11-13T01:35:32.611417Z","iopub.status.idle":"2021-11-13T01:35:32.617764Z","shell.execute_reply.started":"2021-11-13T01:35:32.611365Z","shell.execute_reply":"2021-11-13T01:35:32.617061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im_list=[]\nx_l=[]\ny_l=[]\ngt_x=[]\ngt_y=[]\nwith torch.no_grad():\n    pbar = tqdm(enumerate(visual_loader), total=len(visual_loader))\n    for i_batch, sample_batched in pbar:\n        im = sample_batched['image']\n        h = sample_batched['h']\n        w = sample_batched['w']\n        batch_size = im.size(0)\n        im = im.to(device, dtype=torch.float)\n        im_list.append(im)\n        gt_x.append(sample_batched['gt_x'].numpy().squeeze())\n        gt_y.append(sample_batched['gt_y'].numpy().squeeze())\n        with torch.no_grad():\n            output = net(im)\n            output = output.cpu().numpy()\n            x = output[0][0::2]*240\n            y = output[0][1::2]*180\n            x_l.append(x)\n            y_l.append(y)\nx_l = np.array(x_l)\ny_l = np.array(y_l)\ngt_x = np.array(gt_x)\ngt_y = np.array(gt_y)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:35:34.098182Z","iopub.execute_input":"2021-11-13T01:35:34.099060Z","iopub.status.idle":"2021-11-13T01:35:36.900124Z","shell.execute_reply.started":"2021-11-13T01:35:34.099019Z","shell.execute_reply":"2021-11-13T01:35:36.899316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=80\nplt.imshow(im_list[i].cpu().squeeze(),cmap='gray')\nplt.scatter(x_l[i],y_l[i],label='Prediction')\nplt.scatter(gt_x[i],gt_y[i],label='Ground Truth')\nplt.legend()\nplt.savefig('part2_good1.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:39:50.450595Z","iopub.execute_input":"2021-11-13T01:39:50.450860Z","iopub.status.idle":"2021-11-13T01:39:50.795342Z","shell.execute_reply.started":"2021-11-13T01:39:50.450831Z","shell.execute_reply":"2021-11-13T01:39:50.794693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=50\nplt.imshow(im_list[i].cpu().squeeze(),cmap='gray')\nplt.scatter(x_l[i],y_l[i],label='Prediction')\nplt.scatter(gt_x[i],gt_y[i],label='Ground Truth')\nplt.legend()\nplt.savefig('part2_good2.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:39:51.595395Z","iopub.execute_input":"2021-11-13T01:39:51.596069Z","iopub.status.idle":"2021-11-13T01:39:51.953323Z","shell.execute_reply.started":"2021-11-13T01:39:51.596033Z","shell.execute_reply":"2021-11-13T01:39:51.952706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=62\nplt.imshow(im_list[i].cpu().squeeze(),cmap='gray')\nplt.scatter(x_l[i],y_l[i],label='Prediction')\nplt.scatter(gt_x[i],gt_y[i],label='Ground Truth')\nplt.legend()\nplt.savefig('part2_bad1.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:46:58.656912Z","iopub.execute_input":"2021-11-13T01:46:58.657432Z","iopub.status.idle":"2021-11-13T01:46:59.003323Z","shell.execute_reply.started":"2021-11-13T01:46:58.657396Z","shell.execute_reply":"2021-11-13T01:46:59.002680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#76\ni=78\nplt.imshow(im_list[i].cpu().squeeze(),cmap='gray')\nplt.scatter(x_l[i],y_l[i],label='Prediction')\nplt.scatter(gt_x[i],gt_y[i],label='Ground Truth')\nplt.legend()\nplt.savefig('part2_bad2.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:46:59.004770Z","iopub.execute_input":"2021-11-13T01:46:59.005454Z","iopub.status.idle":"2021-11-13T01:46:59.594934Z","shell.execute_reply.started":"2021-11-13T01:46:59.005419Z","shell.execute_reply":"2021-11-13T01:46:59.594280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_grid(array, ncols=4):\n    index, height, width, channels = array.shape\n    nrows = index//ncols\n    \n    img_grid = (array.reshape(nrows, ncols, height, width, channels)\n              .swapaxes(1,2)\n              .reshape(height*nrows, width*ncols, intensity))\n    \n    return img_grid","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:06:44.922505Z","iopub.execute_input":"2021-11-13T02:06:44.922751Z","iopub.status.idle":"2021-11-13T02:06:44.927388Z","shell.execute_reply.started":"2021-11-13T02:06:44.922724Z","shell.execute_reply":"2021-11-13T02:06:44.926726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualizing filters\n\n#conv1\nconv1_weight = model.conv1.cpu().weight.data.numpy()\nconv1_filter_list = []\nfor i in range(conv1_weight.shape[0]):\n    conv1_filter_list.append(conv1_weight[i,:].squeeze())\nfig = plt.figure(figsize=(6., 6.))\ngrid = ImageGrid(fig, 111, \n                 nrows_ncols=(6, 6), \n                 axes_pad=0.1,  # pad between axes\n                 )\n\nfor ax, im in zip(grid, conv1_filter_list):\n    ax.imshow(im)\ngrid[-1].remove()\ngrid[-2].remove()\ngrid[-3].remove()\ngrid[-4].remove()\n\nplt.savefig('conv1_filters.jpeg')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:09:59.004778Z","iopub.execute_input":"2021-11-13T02:09:59.005537Z","iopub.status.idle":"2021-11-13T02:10:04.483422Z","shell.execute_reply.started":"2021-11-13T02:09:59.005498Z","shell.execute_reply":"2021-11-13T02:10:04.482746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.conv2.cpu().weight.data.numpy()[i,0,:].shape","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:13:30.981231Z","iopub.execute_input":"2021-11-13T02:13:30.981619Z","iopub.status.idle":"2021-11-13T02:13:30.987962Z","shell.execute_reply.started":"2021-11-13T02:13:30.981588Z","shell.execute_reply":"2021-11-13T02:13:30.986995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#conv2\nconv2_weight = model.conv2.cpu().weight.data.numpy()\nconv2_filter_list = []\nfor i in range(conv2_weight.shape[0]):\n    conv2_filter_list.append(conv2_weight[i,0,:].squeeze())\nfig = plt.figure(figsize=(6., 6.))\ngrid = ImageGrid(fig, 111, \n                 nrows_ncols=(8, 8), \n                 axes_pad=0.1,  # pad between axes\n                 )\n\nfor ax, im in zip(grid, conv2_filter_list):\n    ax.imshow(im)\n\nplt.savefig('conv2_filters.jpeg')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:13:35.644780Z","iopub.execute_input":"2021-11-13T02:13:35.645492Z","iopub.status.idle":"2021-11-13T02:13:45.733315Z","shell.execute_reply.started":"2021-11-13T02:13:35.645455Z","shell.execute_reply":"2021-11-13T02:13:45.732670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.conv3.cpu().weight.data.numpy().shape","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:15:29.861301Z","iopub.execute_input":"2021-11-13T02:15:29.861825Z","iopub.status.idle":"2021-11-13T02:15:29.869101Z","shell.execute_reply.started":"2021-11-13T02:15:29.861786Z","shell.execute_reply":"2021-11-13T02:15:29.868319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#conv3\nconv3_weight = model.conv3.cpu().weight.data.numpy()\nconv3_filter_list = []\nfor i in range(conv3_weight.shape[0]):\n    conv3_filter_list.append(conv3_weight[i,0,:].squeeze())\nfig = plt.figure(figsize=(6., 6.))\ngrid = ImageGrid(fig, 111, \n                 nrows_ncols=(6, 6),  \n                 axes_pad=0.1,  # pad between axes\n                 )\ngrid[-1].remove()\ngrid[-2].remove()\ngrid[-3].remove()\ngrid[-4].remove()\nfor ax, im in zip(grid, conv3_filter_list):\n    ax.imshow(im)\n\nplt.savefig('conv3_filters.jpeg')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:16:00.004823Z","iopub.execute_input":"2021-11-13T02:16:00.005564Z","iopub.status.idle":"2021-11-13T02:16:05.554623Z","shell.execute_reply.started":"2021-11-13T02:16:00.005526Z","shell.execute_reply":"2021-11-13T02:16:05.553944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#conv4\nconv4_weight = model.conv4.cpu().weight.data.numpy()\nconv4_filter_list = []\nfor i in range(conv4_weight.shape[0]):\n    conv4_filter_list.append(conv4_weight[i,0,:].squeeze())\nfig = plt.figure(figsize=(6., 6.))\ngrid = ImageGrid(fig, 111, \n                 nrows_ncols=(4, 4),  \n                 axes_pad=0.1,  # pad between axes\n                 )\nfor ax, im in zip(grid, conv4_filter_list):\n    ax.imshow(im)\n\nplt.savefig('conv4_filters.jpeg')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:18:25.904672Z","iopub.execute_input":"2021-11-13T02:18:25.904962Z","iopub.status.idle":"2021-11-13T02:18:29.404512Z","shell.execute_reply.started":"2021-11-13T02:18:25.904932Z","shell.execute_reply":"2021-11-13T02:18:29.403685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#conv5\nconv5_weight = model.conv5.cpu().weight.data.numpy()\nconv5_filter_list = []\nfor i in range(conv5_weight.shape[0]):\n    conv5_filter_list.append(conv5_weight[i,0,:].squeeze())\nfig = plt.figure(figsize=(6., 6.))\ngrid = ImageGrid(fig, 111, \n                 nrows_ncols=(4, 4),  \n                 axes_pad=0.1,  # pad between axes\n                 )\nfor ax, im in zip(grid, conv5_filter_list):\n    ax.imshow(im)\n\nplt.savefig('conv5_filters.jpeg')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T02:19:50.614015Z","iopub.execute_input":"2021-11-13T02:19:50.614290Z","iopub.status.idle":"2021-11-13T02:19:53.161312Z","shell.execute_reply.started":"2021-11-13T02:19:50.614260Z","shell.execute_reply":"2021-11-13T02:19:53.160526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import glob\n\n# files = glob.glob('./*')\n# for f in files:\n#     os.remove(f)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T01:34:27.367426Z","iopub.status.idle":"2021-11-13T01:34:27.367980Z","shell.execute_reply.started":"2021-11-13T01:34:27.367732Z","shell.execute_reply":"2021-11-13T01:34:27.367758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}