{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Implementation from source model\nthis note book will adopt new model call KAN from this [repository](https://github.com/Blealtan/efficient-kan/tree/master/tests)\n- for all kan variance please refer to https://github.com/mintisan/awesome-kan?tab=readme-ov-file\n- for original paper explaination https://github.com/KindXiaoming/pykan?tab=readme-ov-file","metadata":{}},{"cell_type":"code","source":"pip install git+https://github.com/Blealtan/efficient-kan.git","metadata":{"execution":{"iopub.status.busy":"2024-06-08T07:14:56.562821Z","iopub.execute_input":"2024-06-08T07:14:56.563214Z","iopub.status.idle":"2024-06-08T07:17:26.957998Z","shell.execute_reply.started":"2024-06-08T07:14:56.563178Z","shell.execute_reply":"2024-06-08T07:17:26.956854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2024-06-08T05:07:22.764633Z","iopub.execute_input":"2024-06-08T05:07:22.765184Z","iopub.status.idle":"2024-06-08T05:07:41.068800Z","shell.execute_reply.started":"2024-06-08T05:07:22.765137Z","shell.execute_reply":"2024-06-08T05:07:41.067267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficient_kan import KAN","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Train on MNIST\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load MNIST\ntransform = transforms.Compose(\n    [transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))]\n)\ntrainset = torchvision.datasets.MNIST(\n    root=\"./data\", train=True, download=True, transform=transform\n)\nvalset = torchvision.datasets.MNIST(\n    root=\"./data\", train=False, download=True, transform=transform\n)\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:47:13.424060Z","iopub.execute_input":"2024-06-08T08:47:13.424371Z","iopub.status.idle":"2024-06-08T08:47:20.521359Z","shell.execute_reply.started":"2024-06-08T08:47:13.424345Z","shell.execute_reply":"2024-06-08T08:47:20.520255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define model\nmodel = KAN([28 * 28, 64, 10])\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:43:10.417136Z","iopub.status.idle":"2024-06-08T08:43:10.417620Z","shell.execute_reply.started":"2024-06-08T08:43:10.417409Z","shell.execute_reply":"2024-06-08T08:43:10.417428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\n\n# summary(model, (1 , 28*28))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:43:10.419575Z","iopub.status.idle":"2024-06-08T08:43:10.420152Z","shell.execute_reply.started":"2024-06-08T08:43:10.419850Z","shell.execute_reply":"2024-06-08T08:43:10.419883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    # Define optimizer\n    optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n    # Define learning rate scheduler\n    scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n\n    # Define loss\n    criterion = nn.CrossEntropyLoss()\n    for epoch in range(10):\n        # Train\n        model.train()\n        with tqdm(trainloader) as pbar:\n            for i, (images, labels) in enumerate(pbar):\n                images = images.view(-1, 28 * 28).to(device)\n                optimizer.zero_grad()\n                output = model(images)\n                loss = criterion(output, labels.to(device))\n                loss.backward()\n                optimizer.step()\n    #             print(labels)\n                accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n                pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n\n        # Validation\n        model.eval()\n        val_loss = 0\n        val_accuracy = 0\n        with torch.no_grad():\n            for images, labels in valloader:\n                images = images.view(-1, 28 * 28).to(device)\n                output = model(images)\n    #             print(output)\n                val_loss += criterion(output, labels.to(device)).item()\n                val_accuracy += (\n                    (output.argmax(dim=1) == labels.to(device)).float().mean().item()\n                )\n        val_loss /= len(valloader)\n        val_accuracy /= len(valloader)\n\n        # Update learning rate\n        scheduler.step()\n\n        print(\n            f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\"\n        )","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:43:10.422030Z","iopub.status.idle":"2024-06-08T08:43:10.422482Z","shell.execute_reply.started":"2024-06-08T08:43:10.422274Z","shell.execute_reply":"2024-06-08T08:43:10.422291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"load testset","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:32:37.204803Z","iopub.status.idle":"2024-06-08T08:32:37.205255Z","shell.execute_reply.started":"2024-06-08T08:32:37.205043Z","shell.execute_reply":"2024-06-08T08:32:37.205061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testloader = DataLoader(np.array(test).astype(np.float32), batch_size=1, shuffle=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nans = []\nmodel.eval()\nfor i in tqdm(testloader):\n    ans.append(np.argmax(model(i.to(device))[0].cpu().detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:32:37.206763Z","iopub.status.idle":"2024-06-08T08:32:37.207305Z","shell.execute_reply.started":"2024-06-08T08:32:37.207012Z","shell.execute_reply":"2024-06-08T08:32:37.207034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:32:37.208874Z","iopub.status.idle":"2024-06-08T08:32:37.209312Z","shell.execute_reply.started":"2024-06-08T08:32:37.209083Z","shell.execute_reply":"2024-06-08T08:32:37.209099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"Label\"] = ans","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:32:37.211187Z","iopub.status.idle":"2024-06-08T08:32:37.211589Z","shell.execute_reply.started":"2024-06-08T08:32:37.211402Z","shell.execute_reply":"2024-06-08T08:32:37.211418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"Label\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:32:37.213598Z","iopub.status.idle":"2024-06-08T08:32:37.213998Z","shell.execute_reply.started":"2024-06-08T08:32:37.213805Z","shell.execute_reply":"2024-06-08T08:32:37.213820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"sub.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:32:37.215149Z","iopub.status.idle":"2024-06-08T08:32:37.215713Z","shell.execute_reply.started":"2024-06-08T08:32:37.215423Z","shell.execute_reply":"2024-06-08T08:32:37.215447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Implement with resnet concept\ninspired from https://www.kaggle.com/code/shakhrulsiam/implementing-resnet-in-pytorch-from-scratch","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nimport math\nfrom torch import nn\n\n\nclass KANLinear(torch.nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        enable_standalone_scale_spline=True,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        h = (grid_range[1] - grid_range[0]) / grid_size\n        grid = (\n            (\n                torch.arange(-spline_order, grid_size + spline_order + 1) * h\n                + grid_range[0]\n            )\n            .expand(in_features, -1)\n            .contiguous()\n        )\n        self.register_buffer(\"grid\", grid)\n\n        self.base_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n        self.spline_weight = torch.nn.Parameter(\n            torch.Tensor(out_features, in_features, grid_size + spline_order)\n        )\n        if enable_standalone_scale_spline:\n            self.spline_scaler = torch.nn.Parameter(\n                torch.Tensor(out_features, in_features)\n            )\n\n        self.scale_noise = scale_noise\n        self.scale_base = scale_base\n        self.scale_spline = scale_spline\n        self.enable_standalone_scale_spline = enable_standalone_scale_spline\n        self.base_activation = base_activation()\n        self.grid_eps = grid_eps\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        torch.nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n        with torch.no_grad():\n            noise = (\n                (\n                    torch.rand(self.grid_size + 1, self.in_features, self.out_features)\n                    - 1 / 2\n                )\n                * self.scale_noise\n                / self.grid_size\n            )\n            self.spline_weight.data.copy_(\n                (self.scale_spline if not self.enable_standalone_scale_spline else 1.0)\n                * self.curve2coeff(\n                    self.grid.T[self.spline_order : -self.spline_order],\n                    noise,\n                )\n            )\n            if self.enable_standalone_scale_spline:\n                # torch.nn.init.constant_(self.spline_scaler, self.scale_spline)\n                torch.nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n\n    def b_splines(self, x: torch.Tensor):\n        \"\"\"\n        Compute the B-spline bases for the given input tensor.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n\n        Returns:\n            torch.Tensor: B-spline bases tensor of shape (batch_size, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n\n        grid: torch.Tensor = (\n            self.grid\n        )  # (in_features, grid_size + 2 * spline_order + 1)\n        x = x.unsqueeze(-1)\n        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n        for k in range(1, self.spline_order + 1):\n            bases = (\n                (x - grid[:, : -(k + 1)])\n                / (grid[:, k:-1] - grid[:, : -(k + 1)])\n                * bases[:, :, :-1]\n            ) + (\n                (grid[:, k + 1 :] - x)\n                / (grid[:, k + 1 :] - grid[:, 1:(-k)])\n                * bases[:, :, 1:]\n            )\n\n        assert bases.size() == (\n            x.size(0),\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return bases.contiguous()\n\n    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):\n        \"\"\"\n        Compute the coefficients of the curve that interpolates the given points.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n            y (torch.Tensor): Output tensor of shape (batch_size, in_features, out_features).\n\n        Returns:\n            torch.Tensor: Coefficients tensor of shape (out_features, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        assert y.size() == (x.size(0), self.in_features, self.out_features)\n\n        A = self.b_splines(x).transpose(\n            0, 1\n        )  # (in_features, batch_size, grid_size + spline_order)\n        B = y.transpose(0, 1)  # (in_features, batch_size, out_features)\n        solution = torch.linalg.lstsq(\n            A, B\n        ).solution  # (in_features, grid_size + spline_order, out_features)\n        result = solution.permute(\n            2, 0, 1\n        )  # (out_features, in_features, grid_size + spline_order)\n\n        assert result.size() == (\n            self.out_features,\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return result.contiguous()\n\n    @property\n    def scaled_spline_weight(self):\n        return self.spline_weight * (\n            self.spline_scaler.unsqueeze(-1)\n            if self.enable_standalone_scale_spline\n            else 1.0\n        )\n\n    def forward(self, x: torch.Tensor):\n        assert x.size(-1) == self.in_features\n        original_shape = x.shape\n        x = x.view(-1, self.in_features)\n\n        base_output = F.linear(self.base_activation(x), self.base_weight)\n        spline_output = F.linear(\n            self.b_splines(x).view(x.size(0), -1),\n            self.scaled_spline_weight.view(self.out_features, -1),\n        )\n        output = base_output + spline_output\n        \n        output = output.view(*original_shape[:-1], self.out_features)\n        return output\n\n    @torch.no_grad()\n    def update_grid(self, x: torch.Tensor, margin=0.01):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        batch = x.size(0)\n\n        splines = self.b_splines(x)  # (batch, in, coeff)\n        splines = splines.permute(1, 0, 2)  # (in, batch, coeff)\n        orig_coeff = self.scaled_spline_weight  # (out, in, coeff)\n        orig_coeff = orig_coeff.permute(1, 2, 0)  # (in, coeff, out)\n        unreduced_spline_output = torch.bmm(splines, orig_coeff)  # (in, batch, out)\n        unreduced_spline_output = unreduced_spline_output.permute(\n            1, 0, 2\n        )  # (batch, in, out)\n\n        # sort each channel individually to collect data distribution\n        x_sorted = torch.sort(x, dim=0)[0]\n        grid_adaptive = x_sorted[\n            torch.linspace(\n                0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device\n            )\n        ]\n\n        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n        grid_uniform = (\n            torch.arange(\n                self.grid_size + 1, dtype=torch.float32, device=x.device\n            ).unsqueeze(1)\n            * uniform_step\n            + x_sorted[0]\n            - margin\n        )\n\n        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n        grid = torch.concatenate(\n            [\n                grid[:1]\n                - uniform_step\n                * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),\n                grid,\n                grid[-1:]\n                + uniform_step\n                * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),\n            ],\n            dim=0,\n        )\n\n        self.grid.copy_(grid.T)\n        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        \"\"\"\n        Compute the regularization loss.\n\n        This is a dumb simulation of the original L1 regularization as stated in the\n        paper, since the original one requires computing absolutes and entropy from the\n        expanded (batch, in_features, out_features) intermediate tensor, which is hidden\n        behind the F.linear function if we want an memory efficient implementation.\n\n        The L1 regularization is now computed as mean absolute value of the spline\n        weights. The authors implementation also includes this term in addition to the\n        sample-based regularization.\n        \"\"\"\n        l1_fake = self.spline_weight.abs().mean(-1)\n        regularization_loss_activation = l1_fake.sum()\n        p = l1_fake / regularization_loss_activation\n        regularization_loss_entropy = -torch.sum(p * p.log())\n        return (\n            regularize_activation * regularization_loss_activation\n            + regularize_entropy * regularization_loss_entropy\n        )\n\n\nclass KAN(torch.nn.Module):\n    def __init__(\n        self,\n        layers_hidden,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KAN, self).__init__()\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n#         self.softmax = torch.nn.Softmax(dim=1)\n        self.layers = torch.nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                KANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    scale_noise=scale_noise,\n                    scale_base=scale_base,\n                    scale_spline=scale_spline,\n                    base_activation=base_activation,\n                    grid_eps=grid_eps,\n                    grid_range=grid_range,\n                )\n            )\n\n    def forward(self, x: torch.Tensor, update_grid=False):\n        for layer in self.layers:\n            if update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n#         x = self.softmax(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n        )","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:47:30.435645Z","iopub.execute_input":"2024-06-08T08:47:30.436033Z","iopub.status.idle":"2024-06-08T08:47:30.478192Z","shell.execute_reply.started":"2024-06-08T08:47:30.436002Z","shell.execute_reply":"2024-06-08T08:47:30.477244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, in_planes, planes, stride=1, is_last=False):\n        super(BasicBlock, self).__init__()\n        self.is_last = is_last\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != self.expansion * planes:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(self.expansion * planes)\n            )\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        out += self.shortcut(x)\n        preact = out\n        out = F.relu(out)\n        if self.is_last:\n            return out, preact\n        else:\n            return out\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, in_planes, planes, stride=1, is_last=False):\n        super(Bottleneck, self).__init__()\n        self.is_last = is_last\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(self.expansion * planes)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != self.expansion * planes:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(self.expansion * planes)\n            )\n\n    def forward(self, x):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = F.relu(self.bn2(self.conv2(out)))\n        out = self.bn3(self.conv3(out))\n        out += self.shortcut(x)\n        preact = out\n        out = F.relu(out)\n        if self.is_last:\n            return out, preact\n        else:\n            return out\n\nclass ResNet(nn.Module):\n    def __init__(self, block, num_blocks, in_channel=1, zero_init_residual=False):\n        super(ResNet, self).__init__()\n        self.in_planes = 64\n\n        self.conv1 = nn.Conv2d(in_channel, 64, kernel_size=3, stride=1, padding=1,\n                               bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, num_blocks, stride):\n        strides = [stride] + [1] * (num_blocks - 1)\n        layers = []\n        for i in range(num_blocks):\n            stride = strides[i]\n            layers.append(block(self.in_planes, planes, stride))\n            self.in_planes = planes * block.expansion\n        return nn.Sequential(*layers)\n\n    def forward(self, x, layer=100):\n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.layer4(out)\n        out = self.avgpool(out)\n        out = torch.flatten(out, 1)\n        return out\n\ndef resnet18(**kwargs):\n    return ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)\n\ndef resnet34(**kwargs):\n    return ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)\n\ndef resnet50(**kwargs):\n    return ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)\n\ndef resnet101(**kwargs):\n    return ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)\n\nmodel_dict = {\n    'resnet18': [resnet18, 512],\n    'resnet34': [resnet34, 512],\n    'resnet50': [resnet50, 2048],\n    'resnet101': [resnet101, 2048],\n}\n\nclass ResNetModel(nn.Module):\n    \"\"\"backbone + projection head\"\"\"\n    def __init__(self, name='resnet50', head='mlp', feat_dim=128):\n        super(ResNetModel, self).__init__()\n        model_fun, dim_in = model_dict[name]\n        self.layers = []\n        self.encoder = model_fun()\n        if head == 'linear':\n            self.head = nn.Linear(dim_in, feat_dim)\n        elif head == 'mlp':\n            self.head = nn.Sequential(\n                nn.Linear(dim_in, dim_in),\n                nn.ReLU(inplace=True),\n                nn.Linear(dim_in, feat_dim)\n            )\n        elif head == 'kan':\n            self.head = 'kan'\n            layers_hidden = [dim_in, feat_dim]\n            for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n                self.layers.append(\n                    KANLinear(\n                        in_features,\n                        out_features,\n                        grid_size=3,\n                    ).to(device)\n                )\n#             KAN([dim_in ,feat_dim], grid_size = int(dim_in*0.7)\n#             , base_activation=torch.nn.GELU\n        else:\n            raise NotImplementedError(\n                'head not supported: {}'.format(head))\n\n    def forward(self, x, update_grid=False):\n        feat = self.encoder(x)\n        if self.head == 'kan':\n            for layer in self.layers:\n                if update_grid:\n                    layer.update_grid(feat)\n                feat = layer(feat)\n        feat = F.normalize(feat, dim=1)\n        return feat","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:56:33.857594Z","iopub.execute_input":"2024-06-08T08:56:33.857943Z","iopub.status.idle":"2024-06-08T08:56:33.893426Z","shell.execute_reply.started":"2024-06-08T08:56:33.857914Z","shell.execute_reply":"2024-06-08T08:56:33.892588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNetModel(name='resnet18', head='kan', feat_dim=10)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:56:34.416495Z","iopub.execute_input":"2024-06-08T08:56:34.417127Z","iopub.status.idle":"2024-06-08T08:56:36.904328Z","shell.execute_reply.started":"2024-06-08T08:56:34.417097Z","shell.execute_reply":"2024-06-08T08:56:36.903466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:51:39.838465Z","iopub.execute_input":"2024-06-08T08:51:39.838738Z","iopub.status.idle":"2024-06-08T08:51:52.008565Z","shell.execute_reply.started":"2024-06-08T08:51:39.838715Z","shell.execute_reply":"2024-06-08T08:51:52.007575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\nsummary(model, (1, 28, 28))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T08:56:40.872627Z","iopub.execute_input":"2024-06-08T08:56:40.873518Z","iopub.status.idle":"2024-06-08T08:56:40.892822Z","shell.execute_reply.started":"2024-06-08T08:56:40.873485Z","shell.execute_reply":"2024-06-08T08:56:40.891936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define optimizer\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Define learning rate scheduler\nscheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n\n# Define loss\ncriterion = nn.CrossEntropyLoss()\nfor epoch in range(10):\n    # Train\n    model.train()\n    with tqdm(trainloader) as pbar:\n        for i, (images, labels) in enumerate(pbar):\n            images = images.reshape(-1,28,28)\n            images = images.view(-1, 1, 28 , 28).to(device)\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels.to(device))\n            loss.backward()\n            optimizer.step()\n#             print(labels)\n            accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n\n    # Validation\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.reshape(-1,28,28)\n            images = images.view(-1, 1, 28 , 28).to(device)\n            output = model(images)\n#             print(output)\n            val_loss += criterion(output, labels.to(device)).item()\n            val_accuracy += (\n                (output.argmax(dim=1) == labels.to(device)).float().mean().item()\n            )\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n\n    # Update learning rate\n    scheduler.step()\n\n    print(\n        f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\"\n    )","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:00:05.068129Z","iopub.execute_input":"2024-06-08T09:00:05.069105Z","iopub.status.idle":"2024-06-08T09:13:50.002009Z","shell.execute_reply.started":"2024-06-08T09:00:05.069061Z","shell.execute_reply":"2024-06-08T09:13:50.001052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:13:53.924948Z","iopub.execute_input":"2024-06-08T09:13:53.925330Z","iopub.status.idle":"2024-06-08T09:13:56.869129Z","shell.execute_reply.started":"2024-06-08T09:13:53.925298Z","shell.execute_reply":"2024-06-08T09:13:56.868291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testloader = DataLoader(np.array(test).astype(np.float32).reshape(-1,1,28,28), batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:13:56.870561Z","iopub.execute_input":"2024-06-08T09:13:56.870862Z","iopub.status.idle":"2024-06-08T09:13:56.978781Z","shell.execute_reply.started":"2024-06-08T09:13:56.870837Z","shell.execute_reply":"2024-06-08T09:13:56.977999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nans = []\nmodel.eval()\nfor i in tqdm(testloader):\n    ans.append(np.argmax(model(i.to(device))[0].cpu().detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:14:47.616212Z","iopub.execute_input":"2024-06-08T09:14:47.616589Z","iopub.status.idle":"2024-06-08T09:16:42.336656Z","shell.execute_reply.started":"2024-06-08T09:14:47.616560Z","shell.execute_reply":"2024-06-08T09:16:42.335752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\nsub[\"Label\"] = ans\nsub[\"Label\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:16:42.338196Z","iopub.execute_input":"2024-06-08T09:16:42.338498Z","iopub.status.idle":"2024-06-08T09:16:42.454738Z","shell.execute_reply.started":"2024-06-08T09:16:42.338472Z","shell.execute_reply":"2024-06-08T09:16:42.453891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:18:17.005058Z","iopub.execute_input":"2024-06-08T09:18:17.005970Z","iopub.status.idle":"2024-06-08T09:18:17.022143Z","shell.execute_reply.started":"2024-06-08T09:18:17.005936Z","shell.execute_reply":"2024-06-08T09:18:17.021316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"subresnet18kan.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T09:17:06.882100Z","iopub.execute_input":"2024-06-08T09:17:06.882991Z","iopub.status.idle":"2024-06-08T09:17:06.920844Z","shell.execute_reply.started":"2024-06-08T09:17:06.882955Z","shell.execute_reply":"2024-06-08T09:17:06.919900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}