{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Summary\n\n1. Embedding of the last layer of blocks is almost the same if we input random color circles/boxes (similarity of each class centers are >0.95)\n2. In certain range of input, the model can classify highly accurate output. E.g. For the input is constrained in pixel range 192-250, target color (255, 128, 128), the model‘s F1=0.998.\nSo I guess the model causes something like mode corruption when input have color of certain (e.g. lower) pixel values.","metadata":{}},{"cell_type":"code","source":"! pip install nb-black > /dev/null\n! pip install timm > /dev/null\n! pip install torchinfo > /dev/null\n! pip install grad-cam > /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:15:55.698564Z","iopub.execute_input":"2022-09-12T01:15:55.699038Z","iopub.status.idle":"2022-09-12T01:16:29.032801Z","shell.execute_reply.started":"2022-09-12T01:15:55.698985Z","shell.execute_reply":"2022-09-12T01:16:29.031559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! git clone https://github.com/facebookresearch/LeViT.git","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:13:50.393547Z","iopub.execute_input":"2022-09-12T01:13:50.394011Z","iopub.status.idle":"2022-09-12T01:13:52.819244Z","shell.execute_reply.started":"2022-09-12T01:13:50.393952Z","shell.execute_reply":"2022-09-12T01:13:52.818166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\nimport json\nimport requests\nimport sys\n\nimport cv2 as cv\nimport matplotlib.colors as mcolors\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport torch\n\nfrom sklearn import manifold\nfrom torch import nn\nfrom tqdm import tqdm\n\nfrom torchinfo import summary\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\n\nfrom PIL import Image\n\n%load_ext lab_black\n%load_ext autoreload\n%autoreload 2\n\npath = \"./LeViT\"\nif path not in sys.path:\n    sys.path.append(path)\n\ndir_name = \"../input/ai-village-ctf\"\n\nplt.style.use(\"default\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-12T01:31:33.128322Z","iopub.execute_input":"2022-09-12T01:31:33.128771Z","iopub.status.idle":"2022-09-12T01:31:33.217675Z","shell.execute_reply.started":"2022-09-12T01:31:33.128732Z","shell.execute_reply":"2022-09-12T01:31:33.216266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def request_once(data):\n    try:\n        r = requests.post(\"https://crop2.fly.dev/score\", json=data)\n        return eval(r.text)\n    except requests.exceptions.ConnectionError:\n        return {\"error\": \"Connection problems. Contact the CTF organizers.\"}\n\n\ndef make_answer_from_colors(colors: np.ndarray) -> dict:\n    circle_colors, background_colors = (\n        colors[:, :3],\n        colors[:, 3:],\n    )\n    output = {\n        \"circles\": circle_colors.reshape(-1).tolist(),\n        \"squares\": background_colors.reshape(-1).tolist(),\n    }\n    return output\n\n\n# Helper Functions\ndef get_ij(idx):\n    idx = 8 - idx\n    j = idx // 3\n    i = 2 - idx % 3\n    return i, j\n\n\ndef resize(X, size=(224, 224)):\n    X_new = np.array(Image.fromarray(X.astype(np.uint8)).resize(size, resample=3))\n    return X_new\n\n\ndef get_cropped(X, idx):\n    n = 75 * 3\n    i, j = get_ij(idx)\n    crop = resize(X, (n, n))[j * 75 : (j + 1) * 75, i * 75 : (i + 1) * 75, :]\n    return crop\n\n\ndef i2t(img: np.array):\n    return np.array(img, dtype=np.float32) / 255.0\n\n\ndef make_X(x, size=(224, 224)):\n    side = int((len(x) // 3) ** 0.5)\n    x = np.where(x < 255, x, 255)\n    x = np.where(x > 0, x, 0)\n    X_np = x.astype(int).reshape([side, side, 3])\n    X_np = np.array(\n        Image.fromarray(X_np.astype(np.uint8)).resize(size, resample=Image.NEAREST)\n    )\n    return X_np\n\n\ndef pt_ft(X_np):\n    return torch.FloatTensor(X_np)\n\n\ndef score(x, model, X_comp, debug=False):\n    x = (x).astype(int)\n    X_comp = np.divide(X_comp, 0b1010)\n    model.eval()\n    X = make_X(x)\n    print(X.shape)\n    preds = model(pt_ft(i2t(X.T[None])))\n    idx = preds.argmax().item()\n    expected = ((25.5 - X_comp)).flatten()\n    actual_square = get_cropped(X, idx)\n    actual = actual_square.flatten().astype(int)\n    if debug:\n        print({\"idx\": idx})\n        plt.imshow(actual_square)\n        plt.show()\n    sse = ((actual - (expected * 10).astype(int)) ** 2).sum()\n    redness = (actual_square[:, :, 0][actual_square[:, :, 0] > 230] ** 2).sum() + 1\n    if idx == 8:\n        return (1 + preds.max().item()) * sse * redness\n    return sse * redness\n\n\ndef get_submittable(x):\n    return \",\".join([str(int(n)) for n in x])\n\n\nclass ImageGenerator:\n    def __init__(self, anti_aliasing=True):\n        super(ImageGenerator, self).__init__()\n        self.colors = np.random.randint(0, 256, (9, 6)).astype(np.float32)\n        self.anti_aliasing = anti_aliasing\n\n    def generate(self):\n        \"\"\"\n        return:\n            - Tensor (224, 224, 3)\n        \"\"\"\n        tmp_img = np.empty((225, 225, 3), dtype=np.float32)\n        for idx, color in enumerate(self.colors):\n            circle_color, background_color = (\n                color[:3],\n                color[3:],\n            )\n            i, j = get_ij(idx)\n            x = self.gen_block(circle_color.tolist(), background_color.tolist())\n            tmp_img[75 * j : 75 * (j + 1), 75 * i : 75 * (i + 1), :] = x[...]\n\n        img = cv.resize(tmp_img, (224, 224))\n\n        return img\n\n    def gen_block(self, circle_color, background_color):\n        \"\"\"\n        75 x 75の1ブロック分の画像を描画する\n        return:\n            - img: (75, 75, 3)\n        \"\"\"\n        img = np.zeros((75, 75, 3), np.uint8)\n        img[:] = np.array(background_color).reshape(1, 1, 3)\n        line_type = cv.LINE_AA if self.anti_aliasing else cv.LINE_8\n        cv.circle(img, (37, 37), 25, circle_color, -1, lineType=line_type)\n        return img\n\n\nclass ImageGeneratorV2(ImageGenerator):\n    \"\"\"\n    赤い丸(255, 0, 0)を必ず1つ含む画像を生成する。\n    画像は1度にN個生成する。\n    \"\"\"\n\n    def __init__(\n        self,\n        n_instances=100,\n        max_redness=255,\n        anti_aliasing=True,\n        min_pix_value=0,\n        max_pix_value=255,\n        target_color=(255, 0, 0),\n    ):\n        super(ImageGeneratorV2, self).__init__(anti_aliasing=anti_aliasing)\n        self.colors = np.random.randint(\n            min_pix_value, max_pix_value + 1, (n_instances, 9, 6)\n        ).astype(np.float32)\n        target_idxs = np.random.randint(0, 9, (n_instances))\n        self.n_instances = n_instances\n        self.target_idxs = target_idxs\n        self.max_redness = max_redness\n\n        # marking red circles\n        for ins, target_idx in enumerate(target_idxs):\n            # clip the redness of non-target index\n            self.colors[ins, :, 0] = self.colors[ins, target_idx, 0].clip(\n                0, max_redness\n            )\n            self.colors[ins, target_idx, :3] = np.array(target_color).reshape(1, 1, 3)\n\n    def generate(self):\n        \"\"\"\n        return:\n            - Tensor (n_instances, 224, 224, 3)\n        \"\"\"\n        img = np.empty((self.n_instances, 224, 224, 3), dtype=np.float32)\n        for ins, colors in enumerate(self.colors):\n            tmp_img = np.empty((225, 225, 3), dtype=np.float32)\n            for idx, color in enumerate(colors):\n                circle_color, background_color = (\n                    color[:3],\n                    color[3:],\n                )\n                i, j = get_ij(idx)\n                x = self.gen_block(circle_color.tolist(), background_color.tolist())\n                tmp_img[75 * j : 75 * (j + 1), 75 * i : 75 * (i + 1), :] = x[...]\n\n            img[ins] = cv.resize(tmp_img, (224, 224))\n\n        return img, self.target_idxs\n\n\ndef cosine_sim(v, w):\n    return np.matmul(v, w.T) / (\n        np.linalg.norm(v, axis=1, keepdims=True) * np.linalg.norm(w, axis=1)\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:17:58.082810Z","iopub.execute_input":"2022-09-12T01:17:58.083256Z","iopub.status.idle":"2022-09-12T01:17:58.290973Z","shell.execute_reply.started":"2022-09-12T01:17:58.083220Z","shell.execute_reply":"2022-09-12T01:17:58.289814Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = joblib.load(f\"{dir_name}/crop/04JUN_crop_model.pckl\")\nmodel.eval()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-09-12T01:15:12.485094Z","iopub.execute_input":"2022-09-12T01:15:12.485514Z","iopub.status.idle":"2022-09-12T01:15:16.823662Z","shell.execute_reply.started":"2022-09-12T01:15:12.485477Z","shell.execute_reply":"2022-09-12T01:15:16.822551Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model, input_size=(1, 3, 224, 224))","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:16:57.429425Z","iopub.execute_input":"2022-09-12T01:16:57.429833Z","iopub.status.idle":"2022-09-12T01:16:57.724925Z","shell.execute_reply.started":"2022-09-12T01:16:57.429799Z","shell.execute_reply":"2022-09-12T01:16:57.723810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir crop2","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:20:06.639279Z","iopub.execute_input":"2022-09-12T01:20:06.639899Z","iopub.status.idle":"2022-09-12T01:20:07.784197Z","shell.execute_reply.started":"2022-09-12T01:20:06.639844Z","shell.execute_reply":"2022-09-12T01:20:07.782601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment001: Random Input","metadata":{}},{"cell_type":"code","source":"%%time\nn_samples = 1000\nrandom_seed = 1676\n\ny0 = np.empty((n_samples), dtype=int)\ny1 = np.empty((n_samples), dtype=int)\ny_truth = np.empty((n_samples), dtype=int)\nX = np.empty((n_samples, 768))\nX_patch = np.empty((n_samples, 384, 14, 14))\n\nnp.random.seed(random_seed)\n\nwith torch.no_grad():\n    for i in tqdm(range(n_samples)):\n        img_gen = ImageGeneratorV2(n_instances=1, max_redness=255, min_pix_value=0, max_pix_value=255, target_color=(255, 0, 0))\n        imgs, labels = img_gen.generate()\n        \n        x = torch.FloatTensor(imgs).permute(0, 3, 2, 1) / 255.\n        x = model.patch_embed(x)\n        X_patch[i, ...] = x.detach().numpy()\n\n        x = x.flatten(2).transpose(1, 2)\n        x = model.blocks(x)\n        x = x.mean(1)\n        x0, x1 = model.head(x), model.head_dist(x)\n        \n        X[i, :]= x.reshape(-1).detach().numpy()\n        y0[i] = x0.argmax(dim=1).item()\n        y1[i] = x1.argmax(dim=1).item()\n        y_truth[i] = labels[0]\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 6))\nax1.set(title=\"Output distribution of head\")\nax2.set(title=\"Output distribution of dist_head\")\n\nsns.countplot(x=y0, ax=ax1)\nsns.countplot(x=y1, ax=ax2)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:18:35.968857Z","iopub.execute_input":"2022-09-12T01:18:35.969319Z","iopub.status.idle":"2022-09-12T01:20:01.628013Z","shell.execute_reply.started":"2022-09-12T01:18:35.969281Z","shell.execute_reply":"2022-09-12T01:20:01.627043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nrandom_seed = 1676\n\nlinear_models = [\n    PCA(n_components=2),\n    TSNE(\n        n_components=2, perplexity=30, init=\"random\", learning_rate=\"auto\"\n    ),\n]\n\nfig, axes = plt.subplots(2, 3, figsize=(27, 18))\naxes = axes.ravel()\n\nfor i, (linear_model, model_name) in enumerate(zip(linear_models, (\"PCA\", \"t-SNE\"))):\n    X0 = linear_model.fit_transform(X)\n    ax1, ax2, ax3 = axes[3 * i], axes[3* i + 1], axes[3* i + 2]\n\n    for c in np.unique(y_truth):\n        X_ = X0[y0 == c]\n        ax1.scatter(X_[:, 0], X_[:, 1], label=f\"idx={c}\", marker='x', alpha=0.8)\n\n    for c in np.unique(y_truth):\n        X_ = X0[y1 == c]\n        ax2.scatter(X_[:, 0], X_[:, 1], label=f\"idx={c}\", marker='x', alpha=0.8)\n\n    for c in np.unique(y_truth):\n        X_ = X0[y_truth == c]\n        ax3.scatter(X_[:, 0], X_[:, 1], label=f\"idx={c}\", marker='x', alpha=0.8)\n\n    ax1.set(title=f\"Label distribution of Head [{model_name}]\")\n    ax1.legend()\n    ax2.set(title=f\"Label distribution of distillation Head [{model_name}]\")\n    ax2.legend()\n    ax3.set(title=f\"GT Label distribution [{model_name}]\")\n    ax3.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:20:26.267262Z","iopub.execute_input":"2022-09-12T01:20:26.267659Z","iopub.status.idle":"2022-09-12T01:20:32.924188Z","shell.execute_reply.started":"2022-09-12T01:20:26.267625Z","shell.execute_reply":"2022-09-12T01:20:32.923144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import product\n\nwith torch.no_grad():\n    z = torch.FloatTensor(X)\n    z0 = z.mean(dim=0, keepdim=True)\n\n    w1, b1 = model.head.l.parameters()\n    w2, b2 = model.head_dist.l.parameters()\n\nw1, w2, z0 = [x.detach().numpy() for x in (w1, w2, z0)]\nzc = np.empty((9, 768))\nfor c in range(9):\n    zc[c, :] = z[y_truth == c].mean(axis=0).detach().numpy()\n\nfig, axes = plt.subplots(4, 4, figsize=(18, 15))\naxes = axes.ravel()\n\nfor ax, (v1, v2), (n1, n2) in zip(\n    axes,\n    product((w1, w2, zc, z0), (w1, w2, zc, z0)),\n    product((\"w_1\", \"w_2\", \"z_c\", \"z_0\"), (\"w_1\", \"w_2\", \"z_c\", \"z_0\")),\n):\n    if min(v1.shape[0], v2.shape[0]) == 1:\n        ax.bar(\n            x=np.arange(max(v1.shape[0], v2.shape[0])),\n            height=cosine_sim(v1, v2).reshape(-1),\n        )\n    else:\n        sns.heatmap(cosine_sim(v1, v2), ax=ax)\n    ax.set(title=f\"$sim({n1}, {n2})$\")\n\nplt.suptitle(\"Similarity of class center & embeddings\")\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:20:55.091485Z","iopub.execute_input":"2022-09-12T01:20:55.092004Z","iopub.status.idle":"2022-09-12T01:21:02.287034Z","shell.execute_reply.started":"2022-09-12T01:20:55.091949Z","shell.execute_reply":"2022-09-12T01:21:02.285582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots()\nsns.histplot(cosine_sim(z, z0).squeeze(), ax=ax)\nax.set(title=\"Histogram of $sim(z_i, z_0)$\")","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:21:06.428925Z","iopub.execute_input":"2022-09-12T01:21:06.429812Z","iopub.status.idle":"2022-09-12T01:21:06.793863Z","shell.execute_reply.started":"2022-09-12T01:21:06.429757Z","shell.execute_reply":"2022-09-12T01:21:06.791632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axes = plt.subplots(3, 3, figsize=(16, 12), sharex=True)\naxes = axes.ravel()\n\nfor i in range(9):\n    ax = axes[i]\n    sns.histplot(cosine_sim(z, zc[i : i + 1]).squeeze(), ax=ax)\n    ax.set(title=f\"Histogram of $sim(z, z_{i})$\")","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:22:05.024889Z","iopub.execute_input":"2022-09-12T01:22:05.025289Z","iopub.status.idle":"2022-09-12T01:22:07.185770Z","shell.execute_reply.started":"2022-09-12T01:22:05.025256Z","shell.execute_reply":"2022-09-12T01:22:07.184684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_seed = 1676\naverage_type = \"micro\"\n\nwith torch.no_grad():\n    z = torch.FloatTensor(X)\n    z0 = z.mean(dim=0, keepdim=True)\n\n    z1 = model.head(z - z0)\n    y2 = z1.argmax(dim=1)\n\n    z2 = model.head_dist(z - z0)\n    y3 = z2.argmax(dim=1)\n\ny2 = y2.detach().numpy()\ny3 = y3.detach().numpy()\n\nprint(\"=\" * 80)\nprint(\"head\")\nprint(\n    f\"accuracy: {accuracy_score(y_truth, y0):.3f} -> {accuracy_score(y_truth, y2):.3f}\"\n)\nprint(\n    f\"recall: {recall_score(y_truth, y0, average=average_type):.3f} -> {recall_score(y_truth, y2, average=average_type):.3f}\"\n)\nprint(\n    f\"precision: {precision_score(y_truth, y0, average=average_type, zero_division=False):.3f} -> {precision_score(y_truth, y2, average=average_type, zero_division=False):.3f}\"\n)\nprint(\n    f\"F1: {f1_score(y_truth, y0, average=average_type, zero_division=False):.3f} -> {f1_score(y_truth, y2, average=average_type, zero_division=False):.3f}\"\n)\nprint(\"=\" * 80)\nprint(\"head_dist\")\nprint(\n    f\"accuracy: {accuracy_score(y_truth, y1):.3f} -> {accuracy_score(y_truth, y3):.3f}\"\n)\nprint(\n    f\"recall: {recall_score(y_truth, y1, average=average_type):.3f} -> {recall_score(y_truth, y3, average=average_type):.3f}\"\n)\nprint(\n    f\"precision: {precision_score(y_truth, y1, average=average_type, zero_division=False):.3f} -> {precision_score(y_truth, y3, average=average_type, zero_division=False):.3f}\"\n)\nprint(\n    f\"F1: {f1_score(y_truth, y1, average=average_type, zero_division=False):.3f} -> {f1_score(y_truth, y3, average=average_type, zero_division=False):.3f}\"\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:21:32.481608Z","iopub.execute_input":"2022-09-12T01:21:32.482046Z","iopub.status.idle":"2022-09-12T01:21:32.584340Z","shell.execute_reply.started":"2022-09-12T01:21:32.482008Z","shell.execute_reply":"2022-09-12T01:21:32.583437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_grad_cam import GradCAM, XGradCAM\nfrom pytorch_grad_cam.utils.model_targets import (\n    ClassifierOutputTarget,\n    ClassifierOutputSoftmaxTarget,\n)\nfrom pytorch_grad_cam.utils.image import show_cam_on_image\n\n\ndef reshape_transform(tensor, height=4, width=4):\n    result = tensor.reshape(-1, height, width, tensor.size(1))\n\n    # Bring the channels to the first dimension,\n    # like in CNNs.\n    result = result.transpose(2, 3).transpose(1, 2)\n    return result\n\n\nrandom_seed = 1687\nnp.random.seed(random_seed)\nmodel = joblib.load(f\"{dir_name}/crop/04JUN_crop_model.pckl\")\ntarget_class = 0\ntarget_layers = [model._modules[\"blocks\"][-1]._modules[\"m\"][-1]._modules[\"bn\"]]\ncam = XGradCAM(\n    model=model, target_layers=target_layers, reshape_transform=reshape_transform\n)\n\nimg_gen = ImageGeneratorV2(\n    n_instances=50,\n    max_redness=255,\n    min_pix_value=0,\n    max_pix_value=255,\n    target_color=(255, 0, 0),\n)\nrgb_imgs, labels = img_gen.generate()\n\nwith torch.no_grad():\n    input_tensor = torch.FloatTensor(rgb_imgs.transpose(0, 3, 2, 1)) / 255.0\n    out = model(input_tensor)\n    proba = out.softmax(dim=1).detach().numpy()\n    preds = out.argmax(axis=1).detach().numpy()\n\n\nfig, axes = plt.subplots(10, 5 * 2, figsize=(24, 24))\naxes = axes.ravel()\n\nfor i, (y_pred, label, rgb_img) in enumerate(zip(preds, labels, rgb_imgs)):\n    targets = [ClassifierOutputTarget(y_pred)]\n    grayscale_cam = cam(input_tensor=input_tensor[i : i + 1], targets=targets)\n    grayscale_cam = grayscale_cam[0, :]\n    visualization = show_cam_on_image(\n        rgb_img / 255.0 / 4, grayscale_cam.transpose(1, 0), use_rgb=True\n    )  # model の入力はx, yが反転しているので元に戻す必要がある\n\n    ax1, ax2 = axes[2 * i], axes[2 * i + 1]\n    ax1.imshow(rgb_img.astype(np.uint8))\n    ax2.imshow(visualization)\n    ax1.set(title=f\"ogiginal [{label}]\")\n    ax2.set(title=f\"pred [{y_pred}]\")\n    ax1.axis(\"off\")\n    ax2.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:28:14.941438Z","iopub.execute_input":"2022-09-12T01:28:14.943374Z","iopub.status.idle":"2022-09-12T01:28:42.302681Z","shell.execute_reply.started":"2022-09-12T01:28:14.943313Z","shell.execute_reply":"2022-09-12T01:28:42.301584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment002: Constrained Input","metadata":{}},{"cell_type":"markdown","source":"In a certain range of input, the model can classify highly accurately.\n\n* pixel values in 192-255\n* target color = (255, 128, 128)\n\nIn this condition, the model's F1 is 0.998.","metadata":{}},{"cell_type":"code","source":"%%time\nn_samples = 1000\nrandom_seed = 1676\n\ny0 = np.empty((n_samples), dtype=int)\ny1 = np.empty((n_samples), dtype=int)\ny_truth = np.empty((n_samples), dtype=int)\nX = np.empty((n_samples, 768))\nX_patch = np.empty((n_samples, 384, 14, 14))\n\nnp.random.seed(random_seed)\n\nwith torch.no_grad():\n    for i in tqdm(range(n_samples)):\n        img_gen = ImageGeneratorV2(n_instances=1, max_redness=255, min_pix_value=192, max_pix_value=255, target_color=(255, 128, 128))\n        imgs, labels = img_gen.generate()\n        \n        x = torch.FloatTensor(imgs).permute(0, 3, 2, 1) / 255.\n        x = model.patch_embed(x)\n        X_patch[i, ...] = x.detach().numpy()\n\n        x = x.flatten(2).transpose(1, 2)\n        x = model.blocks(x)\n        x = x.mean(1)\n        x0, x1 = model.head(x), model.head_dist(x)\n        \n        X[i, :]= x.reshape(-1).detach().numpy()\n        y0[i] = x0.argmax(dim=1).item()\n        y1[i] = x1.argmax(dim=1).item()\n        y_truth[i] = labels[0]\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 6))\nax1.set(title=\"Output distribution of head\")\nax2.set(title=\"Output distribution of dist_head\")\n\nsns.countplot(x=y0, ax=ax1)\nsns.countplot(x=y1, ax=ax2)\n\nplt.show()\n\n#\n# save results\n#\ndata = {\n    \"y_truth\": y_truth,\n    \"y0\": y0,\n    \"y1\": y1,\n    \"X\": X,\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:23:47.726882Z","iopub.execute_input":"2022-09-12T01:23:47.727339Z","iopub.status.idle":"2022-09-12T01:25:02.322397Z","shell.execute_reply.started":"2022-09-12T01:23:47.727302Z","shell.execute_reply":"2022-09-12T01:25:02.321220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nrandom_seed = 1676\n\nlinear_models = [\n    PCA(n_components=2),\n    TSNE(\n        n_components=2, perplexity=30, init=\"random\", learning_rate=\"auto\"\n    ),\n]\n\nfig, axes = plt.subplots(2, 3, figsize=(27, 18))\naxes = axes.ravel()\n\nfor i, (linear_model, model_name) in enumerate(zip(linear_models, (\"PCA\", \"t-SNE\"))):\n    X0 = linear_model.fit_transform(X)\n    ax1, ax2, ax3 = axes[3 * i], axes[3* i + 1], axes[3* i + 2]\n\n    for c in np.unique(y_truth):\n        X_ = X0[y0 == c]\n        ax1.scatter(X_[:, 0], X_[:, 1], label=f\"idx={c}\", marker='x', alpha=0.8)\n\n    for c in np.unique(y_truth):\n        X_ = X0[y1 == c]\n        ax2.scatter(X_[:, 0], X_[:, 1], label=f\"idx={c}\", marker='x', alpha=0.8)\n\n    for c in np.unique(y_truth):\n        X_ = X0[y_truth == c]\n        ax3.scatter(X_[:, 0], X_[:, 1], label=f\"idx={c}\", marker='x', alpha=0.8)\n\n    ax1.set(title=f\"Label distribution of Head [{model_name}]\")\n    ax1.legend()\n    ax2.set(title=f\"Label distribution of distillation Head [{model_name}]\")\n    ax2.legend()\n    ax3.set(title=f\"GT Label distribution [{model_name}]\")\n    ax3.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:31:39.320471Z","iopub.execute_input":"2022-09-12T01:31:39.320893Z","iopub.status.idle":"2022-09-12T01:31:44.729611Z","shell.execute_reply.started":"2022-09-12T01:31:39.320854Z","shell.execute_reply":"2022-09-12T01:31:44.728526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Both linear and non-linear separability is pretty well.","metadata":{}},{"cell_type":"code","source":"from itertools import product\n\nwith torch.no_grad():\n    z = torch.FloatTensor(X)\n    z0 = z.mean(dim=0, keepdim=True)\n\n    w1, b1 = model.head.l.parameters()\n    w2, b2 = model.head_dist.l.parameters()\n\nw1, w2, z0 = [x.detach().numpy() for x in (w1, w2, z0)]\nzc = np.empty((9, 768))\nfor c in range(9):\n    zc[c, :] = z[y_truth == c].mean(axis=0).detach().numpy()\n\nfig, axes = plt.subplots(4, 4, figsize=(18, 15))\naxes = axes.ravel()\n\nfor ax, (v1, v2), (n1, n2) in zip(\n    axes,\n    product((w1, w2, zc, z0), (w1, w2, zc, z0)),\n    product((\"w_1\", \"w_2\", \"z_c\", \"z_0\"), (\"w_1\", \"w_2\", \"z_c\", \"z_0\")),\n):\n    if min(v1.shape[0], v2.shape[0]) == 1:\n        ax.bar(\n            x=np.arange(max(v1.shape[0], v2.shape[0])),\n            height=cosine_sim(v1, v2).reshape(-1),\n        )\n    else:\n        sns.heatmap(cosine_sim(v1, v2), ax=ax)\n    ax.set(title=f\"$sim({n1}, {n2})$\")\n\nplt.suptitle(\"Similarity of class center & embeddings\")\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:25:07.103037Z","iopub.execute_input":"2022-09-12T01:25:07.103437Z","iopub.status.idle":"2022-09-12T01:25:14.747638Z","shell.execute_reply.started":"2022-09-12T01:25:07.103404Z","shell.execute_reply":"2022-09-12T01:25:14.746121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* at least distillation head (`head_dist`) learns something meaningful, because when constrained input, the similarity of class center (`zc`) and linear weight (`w2`) have high correlations.","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots()\nsns.histplot(cosine_sim(z, z0).squeeze(), ax=ax)\nax.set(title=\"Histogram of $sim(z_i, z_0)$\")","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:25:14.749923Z","iopub.execute_input":"2022-09-12T01:25:14.750319Z","iopub.status.idle":"2022-09-12T01:25:15.093474Z","shell.execute_reply.started":"2022-09-12T01:25:14.750284Z","shell.execute_reply":"2022-09-12T01:25:15.091906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axes = plt.subplots(3, 3, figsize=(16, 12), sharex=True)\naxes = axes.ravel()\n\nfor i in range(9):\n    ax = axes[i]\n    sns.histplot(cosine_sim(z, zc[i : i + 1]).squeeze(), ax=ax)\n    ax.set(title=f\"Histogram of $sim(z, z_{i})$\")","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:25:15.094943Z","iopub.execute_input":"2022-09-12T01:25:15.095346Z","iopub.status.idle":"2022-09-12T01:25:17.443580Z","shell.execute_reply.started":"2022-09-12T01:25:15.095310Z","shell.execute_reply":"2022-09-12T01:25:17.442344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_seed = 1676\naverage_type = \"micro\"\n\nwith torch.no_grad():\n    z = torch.FloatTensor(X)\n    z0 = z.mean(dim=0, keepdim=True)\n\n    z1 = model.head(z - z0)\n    y2 = z1.argmax(dim=1)\n\n    z2 = model.head_dist(z - z0)\n    y3 = z2.argmax(dim=1)\n\ny2 = y2.detach().numpy()\ny3 = y3.detach().numpy()\n\nprint(\"=\" * 80)\nprint(\"head\")\nprint(\n    f\"accuracy: {accuracy_score(y_truth, y0):.3f} -> {accuracy_score(y_truth, y2):.3f}\"\n)\nprint(\n    f\"recall: {recall_score(y_truth, y0, average=average_type):.3f} -> {recall_score(y_truth, y2, average=average_type):.3f}\"\n)\nprint(\n    f\"precision: {precision_score(y_truth, y0, average=average_type, zero_division=False):.3f} -> {precision_score(y_truth, y2, average=average_type, zero_division=False):.3f}\"\n)\nprint(\n    f\"F1: {f1_score(y_truth, y0, average=average_type, zero_division=False):.3f} -> {f1_score(y_truth, y2, average=average_type, zero_division=False):.3f}\"\n)\nprint(\"=\" * 80)\nprint(\"head_dist\")\nprint(\n    f\"accuracy: {accuracy_score(y_truth, y1):.3f} -> {accuracy_score(y_truth, y3):.3f}\"\n)\nprint(\n    f\"recall: {recall_score(y_truth, y1, average=average_type):.3f} -> {recall_score(y_truth, y3, average=average_type):.3f}\"\n)\nprint(\n    f\"precision: {precision_score(y_truth, y1, average=average_type, zero_division=False):.3f} -> {precision_score(y_truth, y3, average=average_type, zero_division=False):.3f}\"\n)\nprint(\n    f\"F1: {f1_score(y_truth, y1, average=average_type, zero_division=False):.3f} -> {f1_score(y_truth, y3, average=average_type, zero_division=False):.3f}\"\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:25:28.010839Z","iopub.execute_input":"2022-09-12T01:25:28.011357Z","iopub.status.idle":"2022-09-12T01:25:28.114513Z","shell.execute_reply.started":"2022-09-12T01:25:28.011310Z","shell.execute_reply":"2022-09-12T01:25:28.113052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_grad_cam import GradCAM, XGradCAM\nfrom pytorch_grad_cam.utils.model_targets import (\n    ClassifierOutputTarget,\n    ClassifierOutputSoftmaxTarget,\n)\nfrom pytorch_grad_cam.utils.image import show_cam_on_image\n\n\ndef reshape_transform(tensor, height=4, width=4):\n    result = tensor.reshape(-1, height, width, tensor.size(1))\n\n    # Bring the channels to the first dimension,\n    # like in CNNs.\n    result = result.transpose(2, 3).transpose(1, 2)\n    return result\n\n\nrandom_seed = 1687\nnp.random.seed(random_seed)\nmodel = joblib.load(f\"{dir_name}/crop/04JUN_crop_model.pckl\")\ntarget_class = 0\ntarget_layers = [model._modules[\"blocks\"][-1]._modules[\"m\"][-1]._modules[\"bn\"]]\ncam = XGradCAM(\n    model=model, target_layers=target_layers, reshape_transform=reshape_transform\n)\n\nimg_gen = ImageGeneratorV2(\n    n_instances=50,\n    max_redness=255,\n    min_pix_value=192,\n    max_pix_value=255,\n    target_color=(255, 128, 128),\n)\nrgb_imgs, labels = img_gen.generate()\n\nwith torch.no_grad():\n    input_tensor = torch.FloatTensor(rgb_imgs.transpose(0, 3, 2, 1)) / 255.0\n    out = model(input_tensor)\n    proba = out.softmax(dim=1).detach().numpy()\n    preds = out.argmax(axis=1).detach().numpy()\n\n\nfig, axes = plt.subplots(10, 5 * 2, figsize=(24, 24))\naxes = axes.ravel()\n\nfor i, (y_pred, label, rgb_img) in enumerate(zip(preds, labels, rgb_imgs)):\n    targets = [ClassifierOutputTarget(y_pred)]\n    grayscale_cam = cam(input_tensor=input_tensor[i : i + 1], targets=targets)\n    grayscale_cam = grayscale_cam[0, :]\n    visualization = show_cam_on_image(\n        rgb_img / 255.0 / 4, grayscale_cam.transpose(1, 0), use_rgb=True\n    )  # model の入力はx, yが反転しているので元に戻す必要がある\n\n    ax1, ax2 = axes[2 * i], axes[2 * i + 1]\n    ax1.imshow(rgb_img.astype(np.uint8))\n    ax2.imshow(visualization)\n    ax1.set(title=f\"ogiginal [{label}]\")\n    ax2.set(title=f\"pred [{y_pred}]\")\n    ax1.axis(\"off\")\n    ax2.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-09-12T01:29:23.103749Z","iopub.execute_input":"2022-09-12T01:29:23.104863Z","iopub.status.idle":"2022-09-12T01:29:42.558233Z","shell.execute_reply.started":"2022-09-12T01:29:23.104808Z","shell.execute_reply":"2022-09-12T01:29:42.557026Z"},"trusted":true},"execution_count":null,"outputs":[]}]}