{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"},{"sourceId":176922142,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Test Model Genaralizability with CCGs\n\n**How?**\n\n- Assign same label (it's 0 in this notebook) for some target numbers (0 and 1 in this notebook), then check whether the patterns of these labels are topologically distinguishable. Further analysis can be developed with other reassignment, however, it is not mentioned here. \n\n*If you're not familiar with the CCGs concept, read [this paper](https://arxiv.org/pdf/2403.05610) and this notebook: https://www.kaggle.com/code/thienanlnguyen/inspiration-towards-topology-ann-manifold-viz.*","metadata":{}},{"cell_type":"code","source":"label2target_dict = {1:0}\nnew_labels = list(range(10))\nfor label in label2target_dict:\n    new_labels[label] = label2target_dict[label]","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:00.446281Z","iopub.execute_input":"2024-11-04T10:19:00.446679Z","iopub.status.idle":"2024-11-04T10:19:00.457398Z","shell.execute_reply.started":"2024-11-04T10:19:00.446636Z","shell.execute_reply":"2024-11-04T10:19:00.456462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")\n\nimport torch\nimport torchvision\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\nfrom torch import optim\nfrom torch.utils.data import Dataset, DataLoader, Subset\n\nfrom tqdm import tqdm\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Running on device:\", DEVICE.upper())","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:01.433359Z","iopub.execute_input":"2024-11-04T10:19:01.433978Z","iopub.status.idle":"2024-11-04T10:19:08.988168Z","shell.execute_reply.started":"2024-11-04T10:19:01.433939Z","shell.execute_reply":"2024-11-04T10:19:08.987184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CIFAR-10 Dataset Loading","metadata":{}},{"cell_type":"code","source":"normalize = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n\ntrain_set = torchvision.datasets.CIFAR10(\n    root=\"./data\", train=True, download=True, transform=normalize)\ntest_set = torchvision.datasets.CIFAR10(\n        root=\"./data\", train=False, download=True, transform=normalize)","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:08.989917Z","iopub.execute_input":"2024-11-04T10:19:08.990345Z","iopub.status.idle":"2024-11-04T10:19:25.445131Z","shell.execute_reply.started":"2024-11-04T10:19:08.990312Z","shell.execute_reply":"2024-11-04T10:19:25.444232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use only 2,048 samples, randomly selected, in the test set for visualization of manifold estimated by artificial neural network (ANN Manifold).","metadata":{}},{"cell_type":"code","source":"train_labels = np.array(new_labels)\n(np.unique(train_labels), len(np.unique(train_labels)))","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:25.446466Z","iopub.execute_input":"2024-11-04T10:19:25.446848Z","iopub.status.idle":"2024-11-04T10:19:25.458217Z","shell.execute_reply.started":"2024-11-04T10:19:25.446806Z","shell.execute_reply":"2024-11-04T10:19:25.457256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VizDataset(Dataset):\n    '''The hidden dataset.'''\n    def __init__(self, dataset, sub_indices):\n        super().__init__()\n        self.dataset = dataset\n        self.sub_indices = sub_indices\n\n    def __len__(self):\n        return len(self.sub_indices)\n\n    def __getitem__(self, idx):\n        example = self.dataset[self.sub_indices[idx]]\n        return example[0], example[1], idx\n    \ntest_labels_split = np.zeros((len(test_set),))\nfor i in range(len(test_set)):\n    test_labels_split[i] = test_set[i][1]\n\nretain_test_idx, compact_test_idx = train_test_split(\n        np.arange(len(test_set)), test_size=2048, shuffle=True, stratify=test_labels_split)\n\nviz_set = VizDataset(test_set, compact_test_idx)","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:25.460511Z","iopub.execute_input":"2024-11-04T10:19:25.461256Z","iopub.status.idle":"2024-11-04T10:19:27.836144Z","shell.execute_reply.started":"2024-11-04T10:19:25.461222Z","shell.execute_reply":"2024-11-04T10:19:27.835285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrained Model Loading","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nfrom torchvision.models import resnet18\n\nIMG_SIZE = [32,32]\nN_CLASSES = 10\n\nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.backbone = nn.Sequential(*list(resnet18(pretrained=True).children())[:-1])\n        self.classifier = nn.Linear(512, N_CLASSES)\n        \n    def forward(self, x):\n        features = self.backbone(x)\n        y = self.classifier(features.reshape(-1,512)).reshape(-1,N_CLASSES)\n        return y","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:27.837182Z","iopub.execute_input":"2024-11-04T10:19:27.837494Z","iopub.status.idle":"2024-11-04T10:19:27.844499Z","shell.execute_reply.started":"2024-11-04T10:19:27.837460Z","shell.execute_reply":"2024-11-04T10:19:27.843586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use resnet50 IMAGENET1K_V1 pretrained weights for a better convergence. ","metadata":{}},{"cell_type":"code","source":"net = Model()\nnet","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:27.845569Z","iopub.execute_input":"2024-11-04T10:19:27.845880Z","iopub.status.idle":"2024-11-04T10:19:28.335066Z","shell.execute_reply.started":"2024-11-04T10:19:27.845847Z","shell.execute_reply":"2024-11-04T10:19:28.334064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cohesive Map Sampling","metadata":{}},{"cell_type":"markdown","source":"*This is from my folked notebook, I leave it here without editing.*\n\nPut your imagination into the process of optimizing a neural network. **Taking two data samples from Science, I felt like I was in space, where the optimization process happened** At first, everything was completely chaotic but I started to notice these two data samples, the only thing that helps me navigate this space. As I was observing the fluctuations of the estimated manifold during the process, Science somehow pulled these two models together. At that moment, I realized that these two points are constantly on the same side relative to the manifold. Something happened in my mind...","metadata":{}},{"cell_type":"code","source":"batch_size = 128\nepochs = 450\nLR = 2e-4\nWD = 1e-2\n\ncohesive_scores = torch.from_numpy(np.zeros((len(viz_set),len(viz_set)))).float()\nnoncohesive_scores = torch.from_numpy(np.zeros((len(viz_set),len(viz_set)))).float()\n    \nnet.to(DEVICE)\ntest_net = Model()\ntest_net.to(DEVICE)\n\nnet.train()\noptimizer = optim.AdamW(net.parameters(), lr=LR, weight_decay=WD)\n\ntest_net.eval()\ntrain_loss = 0.\ntrain_count = 0\nfor epoch in tqdm(range(1,epochs+1)):\n    if (epoch%16)==0:\n        if epoch<(epochs-100):\n            cohesive_scores[:,:] = (cohesive_scores[:,:])*.25\n            noncohesive_scores[:,:] = (noncohesive_scores[:,:])*.25\n        train_loss = train_loss/train_count\n        print(f'training_loss:{train_loss:.5f}')\n        train_loss = 0.\n        train_count = 0\n\n    # Prepare training samples\n    current_train_ids = np.random.choice(range(len(train_set)), size=(1024,), replace=True)\n    train_loader = DataLoader(Subset(train_set, current_train_ids), batch_size=batch_size, shuffle=True)\n    for i, (inputs, targets) in enumerate(train_loader):\n        targets = train_labels[targets.numpy()]\n        test_net.load_state_dict(net.state_dict())\n\n        inputs, targets = inputs.to(DEVICE), torch.from_numpy(targets).to(DEVICE)\n        targets = torch.nn.functional.one_hot(targets, num_classes=10).float()\n\n        optimizer.zero_grad()\n        output = net(inputs)\n        loss = -(torch.log_softmax(output,dim=-1).view(-1,1,10) @ targets.view(-1,10,1)).mean()\n\n        loss.backward()\n        optimizer.step()\n        train_loss = train_loss+loss.item()\n        train_count += 1\n        net.eval()\n        test_net.eval()\n        # cohesive values sampling\n        for _ in range(3):\n            viz_loader = DataLoader(viz_set, batch_size=batch_size, shuffle=True)\n            M = len(viz_loader)\n            indices_list = [None]*M\n            side_indicator_list = [None]*M\n            \n            for j, (inputs, _, indices) in enumerate(viz_loader):\n                inputs, indices = inputs.to(DEVICE), indices.to(DEVICE)\n                with torch.no_grad():\n                    output0 = net(inputs).detach()\n                    output1 = test_net(inputs).detach()\n                    output0 = torch.max(output0,dim=1,keepdim=True).values\n                    output1 = torch.max(output1,dim=1,keepdim=True).values\n                    side_indicator = torch.sign(output0-output1).long().reshape(-1)\n                    indices_list[j] = indices\n                    side_indicator_list[j] = side_indicator\n\n            indices = torch.cat(indices_list,dim=0)\n            side_indicator = torch.cat(side_indicator_list,dim=0)\n            ones = torch.ones(len(side_indicator),len(side_indicator)).long().to(DEVICE)\n            event_indicator = torch.mul(ones,side_indicator.view(1,-1))\n            event_indicator = torch.mul(event_indicator,side_indicator.view(-1,1))\n            r_indices_sq = indices.view(-1,1).expand(len(side_indicator),len(side_indicator))\n            c_indices_sq = indices.view(1,-1).expand(len(side_indicator),len(side_indicator))\n            event_indicator = event_indicator.cpu().reshape(-1)\n            r_indices_sq = r_indices_sq.cpu().reshape(-1)\n            c_indices_sq = c_indices_sq.cpu().reshape(-1)\n\n            cohesive_scores[r_indices_sq,c_indices_sq] = cohesive_scores[r_indices_sq,c_indices_sq]+(event_indicator>0).long()\n            noncohesive_scores[r_indices_sq,c_indices_sq] = noncohesive_scores[r_indices_sq,c_indices_sq]+(event_indicator<0).long()\n        net.train()","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:19:28.337514Z","iopub.execute_input":"2024-11-04T10:19:28.337929Z","iopub.status.idle":"2024-11-04T10:44:34.814387Z","shell.execute_reply.started":"2024-11-04T10:19:28.337885Z","shell.execute_reply":"2024-11-04T10:44:34.812726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*This is from my folked notebook, I leave it here without editing.*\n\nBy using a copy of the neural network as a proxy for the ground truth manifold and performing a one-step training process on the original neural network, I could sample the number of times that any pair of data samples end up in the same side (event A) and the number of times that the pair is on two different sides (event B). Here, for simplicity I took the difference of these two values by assigning values 1 and -1 to events A and B, respectively, and adding them to the corresponding cell for the data pair in the cohesive map (the array cohesive_scores ). Instead of using subtraction, you can use the option of dividing the above total by each other (honestly, I personally prefer to use the division version).\n\n**Udpate: This notebook is about the division version!**","metadata":{}},{"cell_type":"markdown","source":"# Viz & Conclusion","metadata":{}},{"cell_type":"markdown","source":"*This is from my folked notebook, I leave it here without editing.*\n\nAfter sampling the coherence of pairs of data samples (note that the viz_set's labels were not used in the sampling process), with the assumption that **two topologically close samples will have high coherence**, I project the map onto 2D space through the Multidimensional scaling algorithm.","metadata":{}},{"cell_type":"code","source":"from sklearn.manifold import MDS\n \nX = 1.-np.abs((cohesive_scores/(cohesive_scores+noncohesive_scores+.5)).numpy())\nX[range(len(X)),range(len(X))]=0\ny = test_labels_split[compact_test_idx]\n\nprint('Original Dimesnion of X = ', X.shape)\n# Create an MDS model with the desired number of dimensions\n# Number of dimensions for visualization\nn_components = 2 \nmds = MDS(n_components=n_components)\n \n# Fit the MDS model to your data\nX_reduced = mds.fit_transform(X)\n \nprint('Dimesnion of X after MDS = ',X_reduced.shape)\n \n# Visualize the reduced data\nplt.figure(figsize=(8, 6))\nplt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=y, cmap=plt.cm.get_cmap(\"jet\", 10))\nplt.colorbar(label='Class Label', ticks=range(10))\nplt.title(\"MDS Visualization of CIFAR-10 Dataset\")\nplt.xlabel(\"MDS Dimension 1\")\nplt.ylabel(\"MDS Dimension 2\")\nplt.savefig('/kaggle/working/manifold_visualization.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:44:38.835852Z","iopub.execute_input":"2024-11-04T10:44:38.836244Z","iopub.status.idle":"2024-11-04T10:47:38.466039Z","shell.execute_reply.started":"2024-11-04T10:44:38.836207Z","shell.execute_reply":"2024-11-04T10:47:38.465109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*It can be seen in the chart that the above assumption is reasonable*","metadata":{}},{"cell_type":"code","source":"# Visualize the reduced data\nplt.figure(figsize=(8, 6))\nplt.scatter(X_reduced[:, 0], X_reduced[:, 1])\nplt.title(\"MDS Visualization of CIFAR-10 Dataset\")\nplt.xlabel(\"MDS Dimension 1\")\nplt.ylabel(\"MDS Dimension 2\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-04T10:47:38.467781Z","iopub.execute_input":"2024-11-04T10:47:38.468109Z","iopub.status.idle":"2024-11-04T10:47:38.765120Z","shell.execute_reply.started":"2024-11-04T10:47:38.468074Z","shell.execute_reply":"2024-11-04T10:47:38.764168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Without labels, we can distinguish 10 clusters in the above graph. This means that the network generalized the differences between samples across classes instead of just overfitting. In fact, visualizing with or without labels does not affect the conclusion much. As long as there is a topological difference between the labeled sample points, we can draw the same conclusion.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}