{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Please give upvote if you fund this interesting 👍","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        pass\nprint(\"done\")\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-13T08:38:28.151850Z","iopub.execute_input":"2024-06-13T08:38:28.152197Z","iopub.status.idle":"2024-06-13T08:38:29.143192Z","shell.execute_reply.started":"2024-06-13T08:38:28.152170Z","shell.execute_reply":"2024-06-13T08:38:29.142216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:29.145021Z","iopub.execute_input":"2024-06-13T08:38:29.145403Z","iopub.status.idle":"2024-06-13T08:38:34.897426Z","shell.execute_reply.started":"2024-06-13T08:38:29.145377Z","shell.execute_reply":"2024-06-13T08:38:34.896437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Digit Recognizer Data from csv","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import train_test_split\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# Verify the data shapes\nfor images, labels in trainloader:\n    print(images.shape, labels.shape)\n    break\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:34.898600Z","iopub.execute_input":"2024-06-13T08:38:34.899089Z","iopub.status.idle":"2024-06-13T08:38:38.728819Z","shell.execute_reply.started":"2024-06-13T08:38:34.899063Z","shell.execute_reply":"2024-06-13T08:38:38.727908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating kolmogorov arnold network","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nimport math\n\n\nclass KANLinear(torch.nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        enable_standalone_scale_spline=True,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        h = (grid_range[1] - grid_range[0]) / grid_size\n        grid = (\n            (\n                torch.arange(-spline_order, grid_size + spline_order + 1) * h\n                + grid_range[0]\n            )\n            .expand(in_features, -1)\n            .contiguous()\n        )\n        self.register_buffer(\"grid\", grid)\n\n        self.base_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n        self.spline_weight = torch.nn.Parameter(\n            torch.Tensor(out_features, in_features, grid_size + spline_order)\n        )\n        if enable_standalone_scale_spline:\n            self.spline_scaler = torch.nn.Parameter(\n                torch.Tensor(out_features, in_features)\n            )\n\n        self.scale_noise = scale_noise\n        self.scale_base = scale_base\n        self.scale_spline = scale_spline\n        self.enable_standalone_scale_spline = enable_standalone_scale_spline\n        self.base_activation = base_activation()\n        self.grid_eps = grid_eps\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        torch.nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n        with torch.no_grad():\n            noise = (\n                (\n                    torch.rand(self.grid_size + 1, self.in_features, self.out_features)\n                    - 1 / 2\n                )\n                * self.scale_noise\n                / self.grid_size\n            )\n            self.spline_weight.data.copy_(\n                (self.scale_spline if not self.enable_standalone_scale_spline else 1.0)\n                * self.curve2coeff(\n                    self.grid.T[self.spline_order : -self.spline_order],\n                    noise,\n                )\n            )\n            if self.enable_standalone_scale_spline:\n                # torch.nn.init.constant_(self.spline_scaler, self.scale_spline)\n                torch.nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n\n    def b_splines(self, x: torch.Tensor):\n        \"\"\"\n        Compute the B-spline bases for the given input tensor.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n\n        Returns:\n            torch.Tensor: B-spline bases tensor of shape (batch_size, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n\n        grid: torch.Tensor = (\n            self.grid\n        )  # (in_features, grid_size + 2 * spline_order + 1)\n        x = x.unsqueeze(-1)\n        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n        for k in range(1, self.spline_order + 1):\n            bases = (\n                (x - grid[:, : -(k + 1)])\n                / (grid[:, k:-1] - grid[:, : -(k + 1)])\n                * bases[:, :, :-1]\n            ) + (\n                (grid[:, k + 1 :] - x)\n                / (grid[:, k + 1 :] - grid[:, 1:(-k)])\n                * bases[:, :, 1:]\n            )\n\n        assert bases.size() == (\n            x.size(0),\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return bases.contiguous()\n\n    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):\n        \"\"\"\n        Compute the coefficients of the curve that interpolates the given points.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n            y (torch.Tensor): Output tensor of shape (batch_size, in_features, out_features).\n\n        Returns:\n            torch.Tensor: Coefficients tensor of shape (out_features, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        assert y.size() == (x.size(0), self.in_features, self.out_features)\n\n        A = self.b_splines(x).transpose(\n            0, 1\n        )  # (in_features, batch_size, grid_size + spline_order)\n        B = y.transpose(0, 1)  # (in_features, batch_size, out_features)\n        solution = torch.linalg.lstsq(\n            A, B\n        ).solution  # (in_features, grid_size + spline_order, out_features)\n        result = solution.permute(\n            2, 0, 1\n        )  # (out_features, in_features, grid_size + spline_order)\n\n        assert result.size() == (\n            self.out_features,\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return result.contiguous()\n\n    @property\n    def scaled_spline_weight(self):\n        return self.spline_weight * (\n            self.spline_scaler.unsqueeze(-1)\n            if self.enable_standalone_scale_spline\n            else 1.0\n        )\n\n    def forward(self, x: torch.Tensor):\n        assert x.size(-1) == self.in_features\n        original_shape = x.shape\n        x = x.view(-1, self.in_features)\n\n        base_output = F.linear(self.base_activation(x), self.base_weight)\n        spline_output = F.linear(\n            self.b_splines(x).view(x.size(0), -1),\n            self.scaled_spline_weight.view(self.out_features, -1),\n        )\n        output = base_output + spline_output\n        \n        output = output.view(*original_shape[:-1], self.out_features)\n        return output\n\n    @torch.no_grad()\n    def update_grid(self, x: torch.Tensor, margin=0.01):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        batch = x.size(0)\n\n        splines = self.b_splines(x)  # (batch, in, coeff)\n        splines = splines.permute(1, 0, 2)  # (in, batch, coeff)\n        orig_coeff = self.scaled_spline_weight  # (out, in, coeff)\n        orig_coeff = orig_coeff.permute(1, 2, 0)  # (in, coeff, out)\n        unreduced_spline_output = torch.bmm(splines, orig_coeff)  # (in, batch, out)\n        unreduced_spline_output = unreduced_spline_output.permute(\n            1, 0, 2\n        )  # (batch, in, out)\n\n        # sort each channel individually to collect data distribution\n        x_sorted = torch.sort(x, dim=0)[0]\n        grid_adaptive = x_sorted[\n            torch.linspace(\n                0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device\n            )\n        ]\n\n        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n        grid_uniform = (\n            torch.arange(\n                self.grid_size + 1, dtype=torch.float32, device=x.device\n            ).unsqueeze(1)\n            * uniform_step\n            + x_sorted[0]\n            - margin\n        )\n\n        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n        grid = torch.concatenate(\n            [\n                grid[:1]\n                - uniform_step\n                * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),\n                grid,\n                grid[-1:]\n                + uniform_step\n                * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),\n            ],\n            dim=0,\n        )\n\n        self.grid.copy_(grid.T)\n        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        \"\"\"\n        Compute the regularization loss.\n\n        This is a dumb simulation of the original L1 regularization as stated in the\n        paper, since the original one requires computing absolutes and entropy from the\n        expanded (batch, in_features, out_features) intermediate tensor, which is hidden\n        behind the F.linear function if we want an memory efficient implementation.\n\n        The L1 regularization is now computed as mean absolute value of the spline\n        weights. The authors implementation also includes this term in addition to the\n        sample-based regularization.\n        \"\"\"\n        l1_fake = self.spline_weight.abs().mean(-1)\n        regularization_loss_activation = l1_fake.sum()\n        p = l1_fake / regularization_loss_activation\n        regularization_loss_entropy = -torch.sum(p * p.log())\n        return (\n            regularize_activation * regularization_loss_activation\n            + regularize_entropy * regularization_loss_entropy\n        )\n\n\nclass KAN(torch.nn.Module):\n    def __init__(\n        self,\n        layers_hidden,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KAN, self).__init__()\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        self.layers = torch.nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                KANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    scale_noise=scale_noise,\n                    scale_base=scale_base,\n                    scale_spline=scale_spline,\n                    base_activation=base_activation,\n                    grid_eps=grid_eps,\n                    grid_range=grid_range,\n                )\n            )\n\n    def forward(self, x: torch.Tensor, update_grid=False):\n        for layer in self.layers:\n            if update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n        )","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:38.730867Z","iopub.execute_input":"2024-06-13T08:38:38.731144Z","iopub.status.idle":"2024-06-13T08:38:38.771290Z","shell.execute_reply.started":"2024-06-13T08:38:38.731120Z","shell.execute_reply":"2024-06-13T08:38:38.770460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Initialization of network","metadata":{}},{"cell_type":"markdown","source":"Input Layer is 784 i.e. 28 x 28 which is dimension of image <br>\nHidden layer is 2 x input_layer+1 i.e. 1569 <br>\nOutput layer is 10 for all digits","metadata":{}},{"cell_type":"code","source":"# Define model\nmodel = KAN([28 * 28, 28 * 28 * 2 + 1, 10])\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:38.772428Z","iopub.execute_input":"2024-06-13T08:38:38.772947Z","iopub.status.idle":"2024-06-13T08:38:39.440769Z","shell.execute_reply.started":"2024-06-13T08:38:38.772917Z","shell.execute_reply":"2024-06-13T08:38:39.439918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defining Optimizer and Learning Rate Decay","metadata":{}},{"cell_type":"code","source":"# Define optimizer\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Define learning rate scheduler\nscheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:39.441878Z","iopub.execute_input":"2024-06-13T08:38:39.442158Z","iopub.status.idle":"2024-06-13T08:38:39.447197Z","shell.execute_reply.started":"2024-06-13T08:38:39.442134Z","shell.execute_reply":"2024-06-13T08:38:39.446309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define loss\ncriterion = nn.CrossEntropyLoss()\n\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:39.448169Z","iopub.execute_input":"2024-06-13T08:38:39.448442Z","iopub.status.idle":"2024-06-13T08:38:39.457843Z","shell.execute_reply.started":"2024-06-13T08:38:39.448420Z","shell.execute_reply":"2024-06-13T08:38:39.457025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training of Network","metadata":{}},{"cell_type":"code","source":"for epoch in range(20):\n    # Train\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader) as pbar:\n        for i, (images, labels) in enumerate(pbar):\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    # Validation\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    all_labels = []\n    all_preds = []\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            output = model(images)\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_preds.extend(output.argmax(dim=1).cpu().numpy())\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    # Update learning rate\n    scheduler.step()\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss}, Train Accuracy: {epoch_train_accuracy}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:39.458841Z","iopub.execute_input":"2024-06-13T08:38:39.459096Z","iopub.status.idle":"2024-06-13T08:41:33.438495Z","shell.execute_reply.started":"2024-06-13T08:38:39.459075Z","shell.execute_reply":"2024-06-13T08:41:33.437490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training Plot of Network","metadata":{}},{"cell_type":"code","source":"# Plot training and validation loss and accuracy\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Train Accuracy')\nplt.plot(val_accuracies, label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:41:33.439692Z","iopub.execute_input":"2024-06-13T08:41:33.440006Z","iopub.status.idle":"2024-06-13T08:41:33.893442Z","shell.execute_reply.started":"2024-06-13T08:41:33.439964Z","shell.execute_reply":"2024-06-13T08:41:33.892518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion Matrix Plot","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\n# Define class labels\nclass_labels = np.arange(10)\n\n# Confusion Matrix\ncm = confusion_matrix(all_labels, all_preds)\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_labels, yticklabels=class_labels)\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:41:33.895809Z","iopub.execute_input":"2024-06-13T08:41:33.896096Z","iopub.status.idle":"2024-06-13T08:41:34.427667Z","shell.execute_reply.started":"2024-06-13T08:41:33.896071Z","shell.execute_reply":"2024-06-13T08:41:34.426766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creation of submission file","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ntest_data = torch.tensor(test_df.values, dtype=torch.float32)\n\ntest_data = (test_data - 0.5) / 0.5\ntest_data = test_data.to(device)\nmodel.eval()\nwith torch.no_grad():\n    test_output = model(test_data)\n    test_predictions = test_output.argmax(dim=1).cpu().numpy()\n\nsubmission_df = pd.DataFrame({'ImageId': range(1, len(test_predictions) + 1), 'Label': test_predictions})\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:41:34.429031Z","iopub.execute_input":"2024-06-13T08:41:34.429550Z","iopub.status.idle":"2024-06-13T08:41:37.170901Z","shell.execute_reply.started":"2024-06-13T08:41:34.429516Z","shell.execute_reply":"2024-06-13T08:41:37.169916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}