{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Please give upvote if you fund this interesting 👍","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        pass\nprint(\"done\")\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-13T08:38:28.15185Z","iopub.execute_input":"2024-06-13T08:38:28.152197Z","iopub.status.idle":"2024-06-13T08:38:29.143192Z","shell.execute_reply.started":"2024-06-13T08:38:28.15217Z","shell.execute_reply":"2024-06-13T08:38:29.142216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:31:48.732275Z","iopub.execute_input":"2024-08-17T14:31:48.732919Z","iopub.status.idle":"2024-08-17T14:31:48.7393Z","shell.execute_reply.started":"2024-08-17T14:31:48.732882Z","shell.execute_reply":"2024-08-17T14:31:48.738145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import train_test_split\nfrom torchmetrics.classification import MulticlassAccuracy\nfrom tqdm import tqdm\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# Define the HA-KAN model (Assuming you've already implemented the HA_KAN class)\nclass HA_KAN(nn.Module):\n    def __init__(self, layers_hidden, grid_size, spline_order):\n        super(HA_KAN, self).__init__()\n        self.layers_hidden = layers_hidden\n        # Initialize HA-KAN layers here\n        # Example for simplicity (you need to implement according to your network)\n        self.fc1 = nn.Linear(layers_hidden[0], layers_hidden[1])\n        self.fc2 = nn.Linear(layers_hidden[1], layers_hidden[2])\n        self.fc3 = nn.Linear(layers_hidden[2], layers_hidden[3])\n        self.fc4 = nn.Linear(layers_hidden[3], layers_hidden[4])\n        \n    def forward(self, x):\n        x = torch.flatten(x, start_dim=1)\n        x = torch.relu(self.fc1(x))\n        x = torch.relu(self.fc2(x))\n        x = torch.relu(self.fc3(x))\n        x = self.fc4(x)\n        return x\n\n# Initialize the model, loss function, optimizer, and metrics\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = HA_KAN(layers_hidden=[28*28, 1024, 512, 256, 10], grid_size=5, spline_order=3).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nepochs = 10  # For quick training\n\n# Training Loop\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0.0\n    train_acc = MulticlassAccuracy(num_classes=10).to(device)\n\n    with tqdm(trainloader, unit=\"batch\") as tepoch:\n        for inputs, labels in tepoch:\n            tepoch.set_description(f\"Epoch {epoch + 1}\")\n\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n\n            optimizer.step()\n\n            train_loss += loss.item() * inputs.size(0)\n            train_acc.update(outputs, labels)\n\n            tepoch.set_postfix(loss=loss.item(), accuracy=train_acc.compute().item())\n\n    train_loss /= len(trainloader.dataset)\n    train_accuracy = train_acc.compute().item()\n\n    print(f'Epoch {epoch + 1}, Train Loss: {train_loss:.4f}, Train Accuracy: {train_accuracy:.4f}')\n\n    # Evaluation Loop\n    model.eval()\n    val_loss = 0.0\n    val_acc = MulticlassAccuracy(num_classes=10).to(device)\n\n    with torch.no_grad():\n        for inputs, labels in valloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n\n            val_loss += loss.item() * inputs.size(0)\n            val_acc.update(outputs, labels)\n\n    val_loss /= len(valloader.dataset)\n    val_accuracy = val_acc.compute().item()\n\n    print(f'Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:31:50.556668Z","iopub.execute_input":"2024-08-17T14:31:50.557422Z","iopub.status.idle":"2024-08-17T14:33:06.699591Z","shell.execute_reply.started":"2024-08-17T14:31:50.557393Z","shell.execute_reply":"2024-08-17T14:33:06.6987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\n\nclass KANLinear(nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        enable_standalone_scale_spline=True,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        h = (grid_range[1] - grid_range[0]) / grid_size\n        grid = (\n            (\n                torch.arange(-spline_order, grid_size + spline_order + 1) * h\n                + grid_range[0]\n            )\n            .expand(in_features, -1)\n            .contiguous()\n        )\n        self.register_buffer(\"grid\", grid)\n\n        self.base_weight = nn.Parameter(torch.Tensor(out_features, in_features))\n        self.spline_weight = nn.Parameter(\n            torch.Tensor(out_features, in_features, grid_size + spline_order)\n        )\n        if enable_standalone_scale_spline:\n            self.spline_scaler = nn.Parameter(\n                torch.Tensor(out_features, in_features)\n            )\n\n        self.scale_noise = scale_noise\n        self.scale_base = scale_base\n        self.scale_spline = scale_spline\n        self.enable_standalone_scale_spline = enable_standalone_scale_spline\n        self.base_activation = base_activation()\n        self.grid_eps = grid_eps\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n        with torch.no_grad():\n            noise = (\n                (\n                    torch.rand(self.grid_size + 1, self.in_features, self.out_features)\n                    - 1 / 2\n                )\n                * self.scale_noise\n                / self.grid_size\n            )\n            self.spline_weight.data.copy_(\n                (self.scale_spline if not self.enable_standalone_scale_spline else 1.0)\n                * self.curve2coeff(\n                    self.grid.T[self.spline_order : -self.spline_order],\n                    noise,\n                )\n            )\n            if self.enable_standalone_scale_spline:\n                nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n\n    def b_splines(self, x: torch.Tensor):\n        grid = self.grid  \n        x = x.unsqueeze(-1)\n        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n        for k in range(1, self.spline_order + 1):\n            bases = (\n                (x - grid[:, : -(k + 1)])\n                / (grid[:, k:-1] - grid[:, : -(k + 1)])\n                * bases[:, :, :-1]\n            ) + (\n                (grid[:, k + 1 :] - x)\n                / (grid[:, k + 1 :] - grid[:, 1:(-k)])\n                * bases[:, :, 1:]\n            )\n        return bases.contiguous()\n\n    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):\n        A = self.b_splines(x).transpose(\n            0, 1\n        )  \n        B = y.transpose(0, 1)  \n        solution = torch.linalg.lstsq(\n            A, B\n        ).solution  \n        result = solution.permute(\n            2, 0, 1\n        )  \n        return result.contiguous()\n\n    @property\n    def scaled_spline_weight(self):\n        return self.spline_weight * (\n            self.spline_scaler.unsqueeze(-1)\n            if self.enable_standalone_scale_spline\n            else 1.0\n        )\n\n    def forward(self, x: torch.Tensor):\n        original_shape = x.shape\n        x = x.view(-1, self.in_features)\n\n        base_output = F.linear(self.base_activation(x), self.base_weight)\n        spline_output = F.linear(\n            self.b_splines(x).view(x.size(0), -1),\n            self.scaled_spline_weight.view(self.out_features, -1),\n        )\n        output = base_output + spline_output\n        \n        output = output.view(*original_shape[:-1], self.out_features)\n        return output\n\n    @torch.no_grad()\n    def update_grid(self, x: torch.Tensor, margin=0.01):\n        batch = x.size(0)\n        splines = self.b_splines(x).permute(1, 0, 2)  \n        orig_coeff = self.scaled_spline_weight.permute(1, 2, 0)  \n        unreduced_spline_output = torch.bmm(splines, orig_coeff).permute(1, 0, 2)  \n        x_sorted = torch.sort(x, dim=0)[0]\n        grid_adaptive = x_sorted[\n            torch.linspace(\n                0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device\n            )\n        ]\n        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n        grid_uniform = (\n            torch.arange(\n                self.grid_size + 1, dtype=torch.float32, device=x.device\n            ).unsqueeze(1)\n            * uniform_step\n            + x_sorted[0]\n            - margin\n        )\n        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n        grid = torch.cat(\n            [\n                grid[:1]\n                - uniform_step\n                * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),\n                grid,\n                grid[-1:]\n                + uniform_step\n                * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),\n            ],\n            dim=0,\n        )\n\n        self.grid.copy_(grid.T)\n        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        l1_fake = self.spline_weight.abs().mean(-1)\n        regularization_loss_activation = l1_fake.sum()\n        p = l1_fake / regularization_loss_activation\n        regularization_loss_entropy = -torch.sum(p * p.log())\n        return (\n            regularize_activation * regularization_loss_activation\n            + regularize_entropy * regularization_loss_entropy\n        )\n\n\nclass HAKANLinear(KANLinear):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        enable_standalone_scale_spline=True,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n        adaptivity=True,\n    ):\n        super(HAKANLinear, self).__init__(\n            in_features,\n            out_features,\n            grid_size=grid_size,\n            spline_order=spline_order,\n            scale_noise=scale_noise,\n            scale_base=scale_base,\n            scale_spline=scale_spline,\n            enable_standalone_scale_spline=enable_standalone_scale_spline,\n            base_activation=base_activation,\n            grid_eps=grid_eps,\n            grid_range=grid_range,\n        )\n        self.adaptivity = adaptivity\n\n        if self.adaptivity:\n            self.adaptive_layer = nn.Linear(in_features, in_features)\n\n    def forward(self, x: torch.Tensor):\n        if self.adaptivity:\n            x = self.adaptive_layer(x)\n        return super(HAKANLinear, self).forward(x)\n\nclass HAKAN(nn.Module):\n    def __init__(\n        self,\n        layers_hidden,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n        adaptivity=True,\n    ):\n        super(HAKAN, self).__init__()\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n        self.adaptivity = adaptivity\n\n        self.layers = nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                HAKANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    scale_noise=scale_noise,\n                    scale_base=scale_base,\n                    scale_spline=scale_spline,\n                    base_activation=base_activation,\n                    grid_eps=grid_eps,\n                    grid_range=grid_range,\n                    adaptivity=adaptivity,\n                )\n            )\n\n    def forward(self, x: torch.Tensor, update_grid=False):\n        for layer in self.layers:\n            if update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n        )\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T12:34:14.220946Z","iopub.execute_input":"2024-08-17T12:34:14.22131Z","iopub.status.idle":"2024-08-17T12:34:14.261051Z","shell.execute_reply.started":"2024-08-17T12:34:14.221276Z","shell.execute_reply":"2024-08-17T12:34:14.260093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\nfrom tqdm import tqdm\n\n# Assuming you have already defined trainloader, valloader, model, criterion, device\n# Create an instance of the HAKAN model\nmodel = HAKAN(layers_hidden=[28 * 28, 128, 64, 10]).to(device)\n\n# Define optimizer and scheduler\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n\n# Lists to store loss and accuracy values\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\n# Training and validation loop\nfor epoch in range(20):\n    # Train\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader) as pbar:\n        for i, (images, labels) in enumerate(pbar):\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    # Validation\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    all_labels = []\n    all_preds = []\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            output = model(images)\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_preds.extend(output.argmax(dim=1).cpu().numpy())\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    # Update learning rate\n    scheduler.step()\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss:.4f}, Train Accuracy: {epoch_train_accuracy:.4f}, Val Loss: {val_loss:.4f}, Val Accuracy: {val_accuracy:.4f}\")\n\n    ","metadata":{"execution":{"iopub.status.busy":"2024-08-17T12:35:51.010924Z","iopub.execute_input":"2024-08-17T12:35:51.011815Z","iopub.status.idle":"2024-08-17T12:38:58.633939Z","shell.execute_reply.started":"2024-08-17T12:35:51.01178Z","shell.execute_reply":"2024-08-17T12:38:58.633015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot training and validation loss and accuracy\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Train Accuracy')\nplt.plot(val_accuracies, label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T12:39:45.38938Z","iopub.execute_input":"2024-08-17T12:39:45.38977Z","iopub.status.idle":"2024-08-17T12:39:45.846752Z","shell.execute_reply.started":"2024-08-17T12:39:45.389739Z","shell.execute_reply":"2024-08-17T12:39:45.845802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# Define class labels\nclass_labels = np.arange(10)\n\n# Confusion Matrix\ncm = confusion_matrix(all_labels, all_preds)\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_labels, yticklabels=class_labels)\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T12:40:05.902821Z","iopub.execute_input":"2024-08-17T12:40:05.903454Z","iopub.status.idle":"2024-08-17T12:40:06.472276Z","shell.execute_reply.started":"2024-08-17T12:40:05.903422Z","shell.execute_reply":"2024-08-17T12:40:06.471382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ntest_data = torch.tensor(test_df.values, dtype=torch.float32)\n\ntest_data = (test_data - 0.5) / 0.5\ntest_data = test_data.to(device)\nmodel.eval()\nwith torch.no_grad():\n    test_output = model(test_data)\n    test_predictions = test_output.argmax(dim=1).cpu().numpy()\n\nsubmission_df = pd.DataFrame({'ImageId': range(1, len(test_predictions) + 1), 'Label': test_predictions})\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T13:58:27.626361Z","iopub.execute_input":"2024-08-17T13:58:27.626656Z","iopub.status.idle":"2024-08-17T13:58:29.892342Z","shell.execute_reply.started":"2024-08-17T13:58:27.626615Z","shell.execute_reply":"2024-08-17T13:58:29.891378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\n\nclass KANLinear(nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        enable_standalone_scale_spline=True,\n        base_activation=nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n        dropout_prob=0.2\n    ):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n        self.dropout_prob = dropout_prob\n\n        h = (grid_range[1] - grid_range[0]) / grid_size\n        grid = (\n            (\n                torch.arange(-spline_order, grid_size + spline_order + 1) * h\n                + grid_range[0]\n            )\n            .expand(in_features, -1)\n            .contiguous()\n        )\n        self.register_buffer(\"grid\", grid)\n\n        self.base_weight = nn.Parameter(torch.Tensor(out_features, in_features))\n        self.spline_weight = nn.Parameter(\n            torch.Tensor(out_features, in_features, grid_size + spline_order)\n        )\n        if enable_standalone_scale_spline:\n            self.spline_scaler = nn.Parameter(\n                torch.Tensor(out_features, in_features)\n            )\n\n        self.scale_noise = scale_noise\n        self.scale_base = scale_base\n        self.scale_spline = scale_spline\n        self.enable_standalone_scale_spline = enable_standalone_scale_spline\n        self.base_activation = base_activation()\n        self.grid_eps = grid_eps\n\n        self.dropout = nn.Dropout(p=dropout_prob)\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n        with torch.no_grad():\n            noise = (\n                (\n                    torch.rand(self.grid_size + 1, self.in_features, self.out_features)\n                    - 1 / 2\n                )\n                * self.scale_noise\n                / self.grid_size\n            )\n            self.spline_weight.data.copy_(\n                (self.scale_spline if not self.enable_standalone_scale_spline else 1.0)\n                * self.curve2coeff(\n                    self.grid.T[self.spline_order : -self.spline_order],\n                    noise,\n                )\n            )\n            if self.enable_standalone_scale_spline:\n                nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n\n    def b_splines(self, x: torch.Tensor):\n        \"\"\"\n        Compute the B-spline bases for the given input tensor.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n\n        Returns:\n            torch.Tensor: B-spline bases tensor of shape (batch_size, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n\n        grid: torch.Tensor = (\n            self.grid\n        )  # (in_features, grid_size + 2 * spline_order + 1)\n        x = x.unsqueeze(-1)\n        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n        for k in range(1, self.spline_order + 1):\n            bases = (\n                (x - grid[:, : -(k + 1)])\n                / (grid[:, k:-1] - grid[:, : -(k + 1)])\n                * bases[:, :, :-1]\n            ) + (\n                (grid[:, k + 1 :] - x)\n                / (grid[:, k + 1 :] - grid[:, 1:(-k)])\n                * bases[:, :, 1:]\n            )\n\n        assert bases.size() == (\n            x.size(0),\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return bases.contiguous()\n\n    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):\n        \"\"\"\n        Compute the coefficients of the curve that interpolates the given points.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n            y (torch.Tensor): Output tensor of shape (batch_size, in_features, out_features).\n\n        Returns:\n            torch.Tensor: Coefficients tensor of shape (out_features, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        assert y.size() == (x.size(0), self.in_features, self.out_features)\n\n        A = self.b_splines(x).transpose(\n            0, 1\n        )  # (in_features, batch_size, grid_size + spline_order)\n        B = y.transpose(0, 1)  # (in_features, batch_size, out_features)\n        solution = torch.linalg.lstsq(\n            A, B\n        ).solution  # (in_features, grid_size + spline_order, out_features)\n        result = solution.permute(\n            2, 0, 1\n        )  # (out_features, in_features, grid_size + spline_order)\n\n        assert result.size() == (\n            self.out_features,\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return result.contiguous()\n\n    @property\n    def scaled_spline_weight(self):\n        return self.spline_weight * (\n            self.spline_scaler.unsqueeze(-1)\n            if self.enable_standalone_scale_spline\n            else 1.0\n        )\n\n    def forward(self, x: torch.Tensor):\n        assert x.size(-1) == self.in_features\n        original_shape = x.shape\n        x = x.view(-1, self.in_features)\n\n        base_output = F.linear(self.base_activation(x), self.base_weight)\n        spline_output = F.linear(\n            self.b_splines(x).view(x.size(0), -1),\n            self.scaled_spline_weight.view(self.out_features, -1),\n        )\n        output = base_output + spline_output\n        \n        output = self.dropout(output)  # Apply dropout\n        output = output.view(*original_shape[:-1], self.out_features)\n        return output\n\n    @torch.no_grad()\n    def update_grid(self, x: torch.Tensor, margin=0.01):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        batch = x.size(0)\n\n        splines = self.b_splines(x)  # (batch, in, coeff)\n        splines = splines.permute(1, 0, 2)  # (in, batch, coeff)\n        orig_coeff = self.scaled_spline_weight  # (out, in, coeff)\n        orig_coeff = orig_coeff.permute(1, 2, 0)  # (in, coeff, out)\n        unreduced_spline_output = torch.bmm(splines, orig_coeff)  # (in, batch, out)\n        unreduced_spline_output = unreduced_spline_output.permute(\n            1, 0, 2\n        )  # (batch, in, out)\n\n        # sort each channel individually to collect data distribution\n        x_sorted = torch.sort(x, dim=0)[0]\n        grid_adaptive = x_sorted[\n            torch.linspace(\n                0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device\n            )\n        ]\n\n        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n        grid_uniform = (\n            torch.arange(\n                self.grid_size + 1, dtype=torch.float32, device=x.device\n            ).unsqueeze(1)\n            * uniform_step\n            + x_sorted[0]\n            - margin\n        )\n\n        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n        grid = torch.cat(\n            [\n                grid[:1]\n                - uniform_step\n                * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),\n                grid,\n                grid[-1:]\n                + uniform_step\n                * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),\n            ],\n            dim=0,\n        )\n\n        self.grid.copy_(grid.T)\n        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        \"\"\"\n        Compute the regularization loss.\n\n        This is a simulation of the original L1 regularization as stated in the\n        paper, since the original one requires computing absolutes and entropy from the\n        expanded (batch, in_features, out_features) intermediate tensor, which is hidden\n        behind the F.linear function if we want a memory efficient implementation.\n\n        The L1 regularization is now computed as mean absolute value of the spline\n        weights. The authors' implementation also includes this term in addition to the\n        sample-based regularization.\n        \"\"\"\n        l1_fake = self.spline_weight.abs().mean(-1)\n        regularization_loss_activation = l1_fake.sum()\n        p = l1_fake / regularization_loss_activation\n        regularization_loss_entropy = -torch.sum(p * p.log())\n        return (\n            regularize_activation * regularization_loss_activation\n            + regularize_entropy * regularization_loss_entropy\n        )\n\n\nclass HAKAN(nn.Module):\n    def __init__(\n        self,\n        layers_hidden,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        base_activation=nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n        dropout_prob=0.2\n    ):\n        super(HAKAN, self).__init__()\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        self.layers = nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                KANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    scale_noise=scale_noise,\n                    scale_base=scale_base,\n                    scale_spline=scale_spline,\n                    base_activation=base_activation,\n                    grid_eps=grid_eps,\n                    grid_range=grid_range,\n                    dropout_prob=dropout_prob\n                )\n            )\n\n    def forward(self, x: torch.Tensor, update_grid=False):\n        for layer in self.layers:\n            if update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n        )\n\n# Example usage:\nmodel = HAKAN(layers_hidden=[784, 128, 64, 10], dropout_prob=0.5)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.7)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:13:38.839966Z","iopub.execute_input":"2024-08-17T14:13:38.840378Z","iopub.status.idle":"2024-08-17T14:13:38.954056Z","shell.execute_reply.started":"2024-08-17T14:13:38.840349Z","shell.execute_reply":"2024-08-17T14:13:38.952874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import train_test_split\nfrom torchmetrics.classification import MulticlassAccuracy\nfrom tqdm import tqdm\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# Initialize lists for tracking\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\n# Define the device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Initialize model, optimizer, and criterion\nmodel = HAKAN(layers_hidden=[784, 128, 64, 10], dropout_prob=0.5).to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\ncriterion = torch.nn.CrossEntropyLoss()\n\n# Define the scheduler\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')\n\nfor epoch in range(20):\n    # Train\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader, desc=f\"Epoch {epoch+1}\", unit=\"batch\") as pbar:\n        for images, labels in pbar:\n            images = images.to(device)  # Move data to device\n            labels = labels.to(device)  # Move labels to device\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    # Validation\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    all_labels = []\n    all_preds = []\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.to(device)  # Move data to device\n            labels = labels.to(device)  # Move labels to device\n            output = model(images)\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_preds.extend(output.argmax(dim=1).cpu().numpy())\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    # Update learning rate\n    scheduler.step(val_loss)  # Pass the validation loss to the scheduler\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss:.4f}, Train Accuracy: {epoch_train_accuracy:.4f}, Val Loss: {val_loss:.4f}, Val Accuracy: {val_accuracy:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:17:37.424535Z","iopub.execute_input":"2024-08-17T14:17:37.424927Z","iopub.status.idle":"2024-08-17T14:17:40.905442Z","shell.execute_reply.started":"2024-08-17T14:17:37.424899Z","shell.execute_reply":"2024-08-17T14:17:40.904106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# Define KANLinear and HAKAN classes\nclass KANLinear(nn.Module):\n    def __init__(self, in_features, out_features, grid_size, **kwargs):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        # Define parameters here, e.g., weights and biases\n        self.weight = nn.Parameter(torch.randn(in_features, out_features))\n        self.bias = nn.Parameter(torch.randn(out_features))\n\n    def forward(self, x: torch.Tensor):\n        assert x.size(-1) == self.in_features, f\"Expected input size {self.in_features}, but got {x.size(-1)}\"\n        x = x.view(-1, self.in_features)  # Ensure correct reshaping\n        x = torch.matmul(x, self.weight) + self.bias  # Example operation\n        return x\n\n    def regularization_loss(self, regularize_activation=0.8, regularize_entropy=0.8):\n        # Compute regularization loss here\n        return torch.tensor(0.0)  # Replace with actual regularization loss computation\n\nclass HAKAN(nn.Module):\n    def __init__(self, layers_hidden, dropout_prob=0.2, grid_size=5, **kwargs):\n        super(HAKAN, self).__init__()\n        self.layers = nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(KANLinear(in_features, out_features, grid_size, **kwargs))\n            self.layers.append(nn.Dropout(dropout_prob))\n\n    def forward(self, x, update_grid=False):\n        for layer in self.layers:\n            if isinstance(layer, KANLinear) and update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=0.8, regularize_entropy=0.8):\n        return sum(layer.regularization_loss(regularize_activation, regularize_entropy) for layer in self.layers if isinstance(layer, KANLinear))\n\n# Define model\nmodel = HAKAN([28 * 28, 28 * 28 * 2 + 1, 10], grid_size=5, dropout_prob=0.2)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Print model parameters to debug\nfor name, param in model.named_parameters():\n    print(name, param.size())\n\n# Define optimizer\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Define learning rate scheduler\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3, verbose=True)\n# Define loss\ncriterion = nn.CrossEntropyLoss()\n\n# Early Stopping\nearly_stopping_patience = 5\nbest_val_loss = float('inf')\nepochs_no_improve = 0\n\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\n# Training of Network\nfor epoch in range(20):  # Reduced epochs to 20\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader) as pbar:\n        for images, labels in pbar:\n            images = images.to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            output = model(images.view(-1, 28 * 28))\n            loss = criterion(output, labels)\n            loss += model.regularization_loss()  # Add regularization loss\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.to(device)\n            labels = labels.to(device)\n            output = model(images.view(-1, 28 * 28))\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    # Early Stopping Check\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        epochs_no_improve = 0\n    else:\n        epochs_no_improve += 1\n        if epochs_no_improve >= early_stopping_patience:\n            print(f\"Early stopping at epoch {epoch + 1}\")\n            break\n\n    scheduler.step(val_loss)\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss}, Train Accuracy: {epoch_train_accuracy}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\")\n\n# Plotting train/validation loss and accuracy\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.legend()\nplt.title('Loss over epochs')\n\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Train Accuracy')\nplt.plot(val_accuracies, label='Validation Accuracy')\nplt.legend()\nplt.title('Accuracy over epochs')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:20:10.60333Z","iopub.execute_input":"2024-08-17T14:20:10.603688Z","iopub.status.idle":"2024-08-17T14:22:11.108101Z","shell.execute_reply.started":"2024-08-17T14:20:10.603662Z","shell.execute_reply":"2024-08-17T14:22:11.107182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# KANLinear and HAKAN classes (same as before)\n\n# class KANLinear(nn.Module):\n#     # Implementation as before\n\nclass HAKAN(nn.Module):\n    def __init__(self, layers_hidden, dropout_prob=0.2, **kwargs):\n        super(HAKAN, self).__init__()\n        self.layers = nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(KANLinear(in_features, out_features, **kwargs))\n            self.layers.append(nn.Dropout(dropout_prob))\n\n    def forward(self, x, update_grid=False):\n        for layer in self.layers:\n            if isinstance(layer, KANLinear) and update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=0.8, regularize_entropy=0.8):\n        return sum(layer.regularization_loss(regularize_activation, regularize_entropy) for layer in self.layers if isinstance(layer, KANLinear))\n\n# Define model\nmodel = HAKAN([28 * 28, 28 * 28 * 2 + 1, 10], grid_size=5, dropout_prob=0.2)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Define optimizer\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Define learning rate scheduler\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3, verbose=True)\n# Define loss\ncriterion = nn.CrossEntropyLoss()\n\n# Early Stopping\nearly_stopping_patience = 5\nbest_val_loss = float('inf')\nepochs_no_improve = 0\n\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\n# Training of Network\nfor epoch in range(20):  # Reduced epochs to 20\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader) as pbar:\n        for images, labels in pbar:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss += model.regularization_loss()  # Add regularization loss\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            output = model(images)\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    # Early Stopping Check\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        epochs_no_improve = 0\n    else:\n        epochs_no_improve += 1\n        if epochs_no_improve >= early_stopping_patience:\n            print(f\"Early stopping at epoch {epoch + 1}\")\n            break\n\n    scheduler.step(val_loss)\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss}, Train Accuracy: {epoch_train_accuracy}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\")\n\n# Plotting train/validation loss and accuracy\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.legend()\nplt.title('Loss over epochs')\n\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Train Accuracy')\nplt.plot(val_accuracies, label='Validation Accuracy')\nplt.legend()\nplt.title('Accuracy over epochs')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T13:54:55.302539Z","iopub.execute_input":"2024-08-17T13:54:55.302916Z","iopub.status.idle":"2024-08-17T13:58:27.624858Z","shell.execute_reply.started":"2024-08-17T13:54:55.302889Z","shell.execute_reply":"2024-08-17T13:58:27.623902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport seaborn as sns\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# KANLinear and KAN classes (same as before)\n\n# class KANLinear(nn.Module):\n#     # Implementation as before\n\nclass HAKAN(nn.Module):\n    def __init__(self, layers_hidden, dropout_prob=0.3, **kwargs):\n        super(HAKAN, self).__init__()\n        self.layers = nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(KANLinear(in_features, out_features, **kwargs))\n            self.layers.append(nn.Dropout(dropout_prob))\n\n    def forward(self, x, update_grid=False):\n        for layer in self.layers:\n            if isinstance(layer, KANLinear) and update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(layer.regularization_loss(regularize_activation, regularize_entropy) for layer in self.layers if isinstance(layer, KANLinear))\n\n# Define model\nmodel = HAKAN([28 * 28, 28 * 28 * 2 + 1, 10], grid_size=7, dropout_prob=0.4)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Define optimizer\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Define learning rate scheduler\nscheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n# Define loss\ncriterion = nn.CrossEntropyLoss()\n\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\n# Training of Network\nfor epoch in range(30):\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader) as pbar:\n        for images, labels in pbar:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss += model.regularization_loss()  # Add regularization loss\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            output = model(images)\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    scheduler.step()\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss}, Train Accuracy: {epoch_train_accuracy}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\")\n\n# Plotting train/validation loss and accuracy\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.legend()\nplt.title('Loss over epochs')\n\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Train Accuracy')\nplt.plot(val_accuracies, label='Validation Accuracy')\nplt.legend()\nplt.title('Accuracy over epochs')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T13:36:03.270069Z","iopub.execute_input":"2024-08-17T13:36:03.270447Z","iopub.status.idle":"2024-08-17T13:41:39.077313Z","shell.execute_reply.started":"2024-08-17T13:36:03.270418Z","shell.execute_reply":"2024-08-17T13:41:39.076364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import necessary libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\n\n# Define the custom Dataset class for loading MNIST data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load and split the MNIST data\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations and create DataLoaders\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# Verify the data shapes\nfor images, labels in trainloader:\n    print(images.shape, labels.shape)\n    break\n\n# Define the HAKAN architecture\nclass KANLinear(torch.nn.Module):\n    def __init__(self, in_features, out_features, grid_size=5, spline_order=3, \n                 scale_noise=0.1, scale_base=1.0, scale_spline=1.0, \n                 enable_standalone_scale_spline=True, base_activation=torch.nn.SiLU, \n                 grid_eps=0.02, grid_range=[-1, 1]):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        h = (grid_range[1] - grid_range[0]) / grid_size\n        grid = (torch.arange(-spline_order, grid_size + spline_order + 1) * h + grid_range[0]).expand(in_features, -1).contiguous()\n        self.register_buffer(\"grid\", grid)\n\n        self.base_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n        self.spline_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features, grid_size + spline_order))\n        if enable_standalone_scale_spline:\n            self.spline_scaler = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n\n        self.scale_noise = scale_noise\n        self.scale_base = scale_base\n        self.scale_spline = scale_spline\n        self.enable_standalone_scale_spline = enable_standalone_scale_spline\n        self.base_activation = base_activation()\n        self.grid_eps = grid_eps\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        torch.nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n        with torch.no_grad():\n            noise = (torch.rand(self.grid_size + 1, self.in_features, self.out_features) - 1 / 2) * self.scale_noise / self.grid_size\n            self.spline_weight.data.copy_((self.scale_spline if not self.enable_standalone_scale_spline else 1.0) * self.curve2coeff(self.grid.T[self.spline_order:-self.spline_order], noise))\n            if self.enable_standalone_scale_spline:\n                torch.nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n\n    def b_splines(self, x: torch.Tensor):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        grid = self.grid\n        x = x.unsqueeze(-1)\n        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n        for k in range(1, self.spline_order + 1):\n            bases = ((x - grid[:, :-(k + 1)]) / (grid[:, k:-1] - grid[:, :-(k + 1)]) * bases[:, :, :-1]) + \\\n                    ((grid[:, k + 1:] - x) / (grid[:, k + 1:] - grid[:, 1:-k]) * bases[:, :, 1:])\n        return bases.contiguous()\n\n    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        assert y.size() == (x.size(0), self.in_features, self.out_features)\n\n        A = self.b_splines(x).transpose(0, 1)\n        B = y.transpose(0, 1)\n        solution = torch.linalg.lstsq(A, B).solution\n        result = solution.permute(2, 0, 1)\n        return result.contiguous()\n\n    @property\n    def scaled_spline_weight(self):\n        return self.spline_weight * (self.spline_scaler.unsqueeze(-1) if self.enable_standalone_scale_spline else 1.0)\n\n    def forward(self, x: torch.Tensor):\n        assert x.size(-1) == self.in_features\n        original_shape = x.shape\n        x = x.view(-1, self.in_features)\n\n        base_output = F.linear(self.base_activation(x), self.base_weight)\n        spline_output = F.linear(self.b_splines(x).view(x.size(0), -1), self.scaled_spline_weight.view(self.out_features, -1))\n        output = base_output + spline_output\n        return output.view(*original_shape[:-1], self.out_features)\n\n    @torch.no_grad()\n    def update_grid(self, x: torch.Tensor, margin=0.01):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        batch = x.size(0)\n\n        splines = self.b_splines(x).permute(1, 0, 2)\n        orig_coeff = self.scaled_spline_weight.permute(1, 2, 0)\n        unreduced_spline_output = torch.bmm(splines, orig_coeff).permute(1, 0, 2)\n\n        x_sorted = torch.sort(x, dim=0)[0]\n        grid_adaptive = x_sorted[torch.linspace(0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device)]\n        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n        grid_uniform = torch.arange(self.grid_size + 1, dtype=torch.float32, device=x.device).unsqueeze(1) * uniform_step + x_sorted[0] - margin\n\n        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n        grid = torch.concatenate([grid[:1] - uniform_step * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),\n                                  grid,\n                                  grid[-1:] + uniform_step * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1)], dim=0)\n\n        self.grid.copy_(grid.T)\n        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        l1_fake = self.spline_weight.abs().mean(-1)\n        regularization_loss_activation = l1_fake.sum()\n        p = l1_fake / regularization_loss_activation\n        regularization_loss_entropy = -torch.sum(p * p.log())\n        return (regularize_activation * regularization_loss_activation + regularize_entropy * regularization_loss_entropy)\n\n\nclass HAKAN(torch.nn.Module):\n    def __init__(self, layers_hidden, grid_size=5, spline_order=3, scale_noise=0.1,\n                 scale_base=1.0, scale_spline=1.0, base_activation=torch.nn.SiLU,\n                 grid_eps=0.02, grid_range=[-1, 1]):\n        super(HAKAN, self).__init__()\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        self.layers = torch.nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(KANLinear(in_features, out_features, grid_size=grid_size, spline_order=spline_order,\n                                         scale_noise=scale_noise, scale_base=scale_base,\n                                         scale_spline=scale_spline, base_activation=base_activation,\n                                         grid_eps=grid_eps, grid_range=grid_range))\n\n    def forward(self, x: torch.Tensor):\n        for layer in self.layers[:-1]:\n            x = torch.relu(layer(x))\n        x = torch.sigmoid(self.layers[-1](x))\n        return x\n\n# Define the model, optimizer, and loss function\nmodel = HAKAN(layers_hidden=[784, 128, 64, 10], grid_size=5, spline_order=3)\noptimizer = optim.Adam(model.parameters(), lr=0.001)\ncriterion = nn.CrossEntropyLoss()\n\n# Train the HAKAN model\ndef train_hakan_model(model, trainloader, valloader, optimizer, criterion, epochs=5):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = model.to(device)\n    train_losses = []\n    val_losses = []\n    accuracy_history = []\n\n    for epoch in range(epochs):\n        model.train()\n        running_loss = 0.0\n        correct = 0\n        total = 0\n\n        for inputs, labels in tqdm(trainloader):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs.view(inputs.size(0), -1))\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n        epoch_loss = running_loss / len(trainloader)\n        epoch_acc = correct / total\n        train_losses.append(epoch_loss)\n        accuracy_history.append(epoch_acc)\n\n        model.eval()\n        val_loss = 0.0\n        val_correct = 0\n        val_total = 0\n        with torch.no_grad():\n            for inputs, labels in valloader:\n                inputs, labels = inputs.to(device), labels.to(device)\n                outputs = model(inputs.view(inputs.size(0), -1))\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n                _, val_predicted = torch.max(outputs.data, 1)\n                val_total += labels.size(0)\n                val_correct += (val_predicted == labels).sum().item()\n\n        val_loss /= len(valloader)\n        val_losses.append(val_loss)\n        val_acc = val_correct / val_total\n\n        print(f'Epoch [{epoch + 1}/{epochs}], Loss: {epoch_loss:.4f}, Accuracy: {epoch_acc:.4f}, Val Loss: {val_loss:.4f}, Val Accuracy: {val_acc:.4f}')\n\n        # Track progress with additional information\n        print(f'Epoch [{epoch + 1}/{epochs}] completed.')\n\n    # Plot training and validation losses\n    plt.figure(figsize=(10, 4))\n    plt.plot(train_losses, label='Training Loss')\n    plt.plot(val_losses, label='Validation Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.show()\n\n    return train_losses, val_losses, accuracy_history\n\ntrain_losses, val_losses, accuracy_history = train_hakan_model(model, trainloader, valloader, optimizer, criterion, epochs=25)\n\n# Generate a classification report and confusion matrix\ndef evaluate_model(model, valloader):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model.eval()\n    y_true = []\n    y_pred = []\n    with torch.no_grad():\n        for inputs, labels in valloader:\n            inputs = inputs.to(device)\n            outputs = model(inputs.view(inputs.size(0), -1))\n            _, predicted = torch.max(outputs.data, 1)\n            y_true.extend(labels.tolist())\n            y_pred.extend(predicted.cpu().tolist())\n\n    # Confusion matrix\n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(10, 8))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    plt.show()\n\nevaluate_model(model, valloader)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T12:44:14.8316Z","iopub.execute_input":"2024-08-17T12:44:14.832067Z","iopub.status.idle":"2024-08-17T12:47:40.608786Z","shell.execute_reply.started":"2024-08-17T12:44:14.832039Z","shell.execute_reply":"2024-08-17T12:47:40.607927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Digit Recognizer Data from csv","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import train_test_split\n\n# Custom Dataset class to handle the data\nclass MNISTDataset(Dataset):\n    def __init__(self, data, labels, transform=None):\n        self.data = data\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        image = self.data[idx].astype('float32').reshape(28, 28) / 255.0\n        label = self.labels[idx]\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Load the MNIST data from CSV\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\n# Split the data into features and targets\nX = df.iloc[:, 1:].values\ny = df.iloc[:, 0].values\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Create Dataset objects\ntrainset = MNISTDataset(X_train, y_train, transform=transform)\nvalset = MNISTDataset(X_val, y_val, transform=transform)\n\n# Create DataLoaders\ntrainloader = DataLoader(trainset, batch_size=64, shuffle=True)\nvalloader = DataLoader(valset, batch_size=64, shuffle=False)\n\n# Verify the data shapes\nfor images, labels in trainloader:\n    print(images.shape, labels.shape)\n    break\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:33:06.701721Z","iopub.execute_input":"2024-08-17T14:33:06.702083Z","iopub.status.idle":"2024-08-17T14:33:09.584186Z","shell.execute_reply.started":"2024-08-17T14:33:06.702049Z","shell.execute_reply":"2024-08-17T14:33:09.583149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating kolmogorov arnold network","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nimport math\n\n\nclass KANLinear(torch.nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        enable_standalone_scale_spline=True,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KANLinear, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        h = (grid_range[1] - grid_range[0]) / grid_size\n        grid = (\n            (\n                torch.arange(-spline_order, grid_size + spline_order + 1) * h\n                + grid_range[0]\n            )\n            .expand(in_features, -1)\n            .contiguous()\n        )\n        self.register_buffer(\"grid\", grid)\n\n        self.base_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n        self.spline_weight = torch.nn.Parameter(\n            torch.Tensor(out_features, in_features, grid_size + spline_order)\n        )\n        if enable_standalone_scale_spline:\n            self.spline_scaler = torch.nn.Parameter(\n                torch.Tensor(out_features, in_features)\n            )\n\n        self.scale_noise = scale_noise\n        self.scale_base = scale_base\n        self.scale_spline = scale_spline\n        self.enable_standalone_scale_spline = enable_standalone_scale_spline\n        self.base_activation = base_activation()\n        self.grid_eps = grid_eps\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        torch.nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n        with torch.no_grad():\n            noise = (\n                (\n                    torch.rand(self.grid_size + 1, self.in_features, self.out_features)\n                    - 1 / 2\n                )\n                * self.scale_noise\n                / self.grid_size\n            )\n            self.spline_weight.data.copy_(\n                (self.scale_spline if not self.enable_standalone_scale_spline else 1.0)\n                * self.curve2coeff(\n                    self.grid.T[self.spline_order : -self.spline_order],\n                    noise,\n                )\n            )\n            if self.enable_standalone_scale_spline:\n                # torch.nn.init.constant_(self.spline_scaler, self.scale_spline)\n                torch.nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n\n    def b_splines(self, x: torch.Tensor):\n        \"\"\"\n        Compute the B-spline bases for the given input tensor.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n\n        Returns:\n            torch.Tensor: B-spline bases tensor of shape (batch_size, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n\n        grid: torch.Tensor = (\n            self.grid\n        )  # (in_features, grid_size + 2 * spline_order + 1)\n        x = x.unsqueeze(-1)\n        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n        for k in range(1, self.spline_order + 1):\n            bases = (\n                (x - grid[:, : -(k + 1)])\n                / (grid[:, k:-1] - grid[:, : -(k + 1)])\n                * bases[:, :, :-1]\n            ) + (\n                (grid[:, k + 1 :] - x)\n                / (grid[:, k + 1 :] - grid[:, 1:(-k)])\n                * bases[:, :, 1:]\n            )\n\n        assert bases.size() == (\n            x.size(0),\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return bases.contiguous()\n\n    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):\n        \"\"\"\n        Compute the coefficients of the curve that interpolates the given points.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, in_features).\n            y (torch.Tensor): Output tensor of shape (batch_size, in_features, out_features).\n\n        Returns:\n            torch.Tensor: Coefficients tensor of shape (out_features, in_features, grid_size + spline_order).\n        \"\"\"\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        assert y.size() == (x.size(0), self.in_features, self.out_features)\n\n        A = self.b_splines(x).transpose(\n            0, 1\n        )  # (in_features, batch_size, grid_size + spline_order)\n        B = y.transpose(0, 1)  # (in_features, batch_size, out_features)\n        solution = torch.linalg.lstsq(\n            A, B\n        ).solution  # (in_features, grid_size + spline_order, out_features)\n        result = solution.permute(\n            2, 0, 1\n        )  # (out_features, in_features, grid_size + spline_order)\n\n        assert result.size() == (\n            self.out_features,\n            self.in_features,\n            self.grid_size + self.spline_order,\n        )\n        return result.contiguous()\n\n    @property\n    def scaled_spline_weight(self):\n        return self.spline_weight * (\n            self.spline_scaler.unsqueeze(-1)\n            if self.enable_standalone_scale_spline\n            else 1.0\n        )\n\n    def forward(self, x: torch.Tensor):\n        assert x.size(-1) == self.in_features\n        original_shape = x.shape\n        x = x.view(-1, self.in_features)\n\n        base_output = F.linear(self.base_activation(x), self.base_weight)\n        spline_output = F.linear(\n            self.b_splines(x).view(x.size(0), -1),\n            self.scaled_spline_weight.view(self.out_features, -1),\n        )\n        output = base_output + spline_output\n        \n        output = output.view(*original_shape[:-1], self.out_features)\n        return output\n\n    @torch.no_grad()\n    def update_grid(self, x: torch.Tensor, margin=0.01):\n        assert x.dim() == 2 and x.size(1) == self.in_features\n        batch = x.size(0)\n\n        splines = self.b_splines(x)  # (batch, in, coeff)\n        splines = splines.permute(1, 0, 2)  # (in, batch, coeff)\n        orig_coeff = self.scaled_spline_weight  # (out, in, coeff)\n        orig_coeff = orig_coeff.permute(1, 2, 0)  # (in, coeff, out)\n        unreduced_spline_output = torch.bmm(splines, orig_coeff)  # (in, batch, out)\n        unreduced_spline_output = unreduced_spline_output.permute(\n            1, 0, 2\n        )  # (batch, in, out)\n\n        # sort each channel individually to collect data distribution\n        x_sorted = torch.sort(x, dim=0)[0]\n        grid_adaptive = x_sorted[\n            torch.linspace(\n                0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device\n            )\n        ]\n\n        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n        grid_uniform = (\n            torch.arange(\n                self.grid_size + 1, dtype=torch.float32, device=x.device\n            ).unsqueeze(1)\n            * uniform_step\n            + x_sorted[0]\n            - margin\n        )\n\n        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n        grid = torch.concatenate(\n            [\n                grid[:1]\n                - uniform_step\n                * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),\n                grid,\n                grid[-1:]\n                + uniform_step\n                * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),\n            ],\n            dim=0,\n        )\n\n        self.grid.copy_(grid.T)\n        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        \"\"\"\n        Compute the regularization loss.\n\n        This is a dumb simulation of the original L1 regularization as stated in the\n        paper, since the original one requires computing absolutes and entropy from the\n        expanded (batch, in_features, out_features) intermediate tensor, which is hidden\n        behind the F.linear function if we want an memory efficient implementation.\n\n        The L1 regularization is now computed as mean absolute value of the spline\n        weights. The authors implementation also includes this term in addition to the\n        sample-based regularization.\n        \"\"\"\n        l1_fake = self.spline_weight.abs().mean(-1)\n        regularization_loss_activation = l1_fake.sum()\n        p = l1_fake / regularization_loss_activation\n        regularization_loss_entropy = -torch.sum(p * p.log())\n        return (\n            regularize_activation * regularization_loss_activation\n            + regularize_entropy * regularization_loss_entropy\n        )\n\n\nclass KAN(torch.nn.Module):\n    def __init__(\n        self,\n        layers_hidden,\n        grid_size=5,\n        spline_order=3,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        base_activation=torch.nn.SiLU,\n        grid_eps=0.02,\n        grid_range=[-1, 1],\n    ):\n        super(KAN, self).__init__()\n        self.grid_size = grid_size\n        self.spline_order = spline_order\n\n        self.layers = torch.nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                KANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    scale_noise=scale_noise,\n                    scale_base=scale_base,\n                    scale_spline=scale_spline,\n                    base_activation=base_activation,\n                    grid_eps=grid_eps,\n                    grid_range=grid_range,\n                )\n            )\n\n    def forward(self, x: torch.Tensor, update_grid=False):\n        for layer in self.layers:\n            if update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n        )","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:33:09.585836Z","iopub.execute_input":"2024-08-17T14:33:09.586382Z","iopub.status.idle":"2024-08-17T14:33:09.631345Z","shell.execute_reply.started":"2024-08-17T14:33:09.586344Z","shell.execute_reply":"2024-08-17T14:33:09.630543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\n\nclass HAKAN(nn.Module):\n    def __init__(self, layers_hidden, grid_size=5, spline_order=3, dropout_prob=0.2, **kwargs):\n        super(HAKAN, self).__init__()\n        self.layers = nn.ModuleList()\n        self.dropout = nn.Dropout(dropout_prob)\n        \n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                KANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    **kwargs\n                )\n            )\n        \n        # Adding Batch Normalization and Residual Connections\n        self.batch_norm_layers = nn.ModuleList([\n            nn.BatchNorm1d(out_features) for in_features, out_features in zip(layers_hidden[:-1], layers_hidden[1:])\n        ])\n        \n        # Residual connections - need to match dimensions for addition\n        self.residual_layers = nn.ModuleList([\n            nn.Sequential(\n                nn.Linear(in_features, out_features, bias=False),\n                nn.BatchNorm1d(out_features)\n            ) for in_features, out_features in zip(layers_hidden[:-1], layers_hidden[1:])\n        ])\n\n    def forward(self, x, update_grid=False):\n        for i, layer in enumerate(self.layers):\n            if update_grid:\n                layer.update_grid(x)\n            \n            # Apply KANLinear layer\n            residual = x\n            x = layer(x)\n            \n            # Apply Batch Normalization and Dropout\n            x = self.batch_norm_layers[i](x)\n            x = F.relu(x)\n            x = self.dropout(x)\n            \n            # Add Residual Connection\n            residual = self.residual_layers[i](residual)\n            x += residual\n        \n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n        )\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:33:09.633675Z","iopub.execute_input":"2024-08-17T14:33:09.633996Z","iopub.status.idle":"2024-08-17T14:33:09.646976Z","shell.execute_reply.started":"2024-08-17T14:33:09.633969Z","shell.execute_reply":"2024-08-17T14:33:09.645919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass HAKAN(nn.Module):\n    def __init__(\n        self,\n        layers_hidden,\n        grid_size=5,\n        spline_order=3,\n        dropout_prob=0.2,\n        scale_noise=0.1,\n        scale_base=1.0,\n        scale_spline=1.0,\n        base_activation=nn.SiLU,\n        grid_eps=0.02\n    ):\n        super(HAKAN, self).__init__()\n        self.layers = nn.ModuleList()\n        for in_features, out_features in zip(layers_hidden, layers_hidden[1:]):\n            self.layers.append(\n                KANLinear(\n                    in_features,\n                    out_features,\n                    grid_size=grid_size,\n                    spline_order=spline_order,\n                    scale_noise=scale_noise,\n                    scale_base=scale_base,\n                    scale_spline=scale_spline,\n                    base_activation=base_activation,\n                    grid_eps=grid_eps\n                )\n            )\n            # Add dropout after each layer\n            self.layers.append(nn.Dropout(dropout_prob))\n\n    def forward(self, x: torch.Tensor, update_grid=False):\n        for layer in self.layers:\n            if isinstance(layer, KANLinear) and update_grid:\n                layer.update_grid(x)\n            x = layer(x)\n        return x\n\n    def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n        return sum(\n            layer.regularization_loss(regularize_activation, regularize_entropy)\n            for layer in self.layers\n            if isinstance(layer, KANLinear)\n        )\n\n# Define model\nmodel = HAKAN(\n    layers_hidden=[28 * 28, 28 * 28 * 2 + 1, 10],\n    grid_size=5,\n    spline_order=3,\n    dropout_prob=0.5,  # Increased dropout probability\n    scale_noise=0.1,\n    scale_base=1.0,\n    scale_spline=1.0,\n    base_activation=nn.SiLU,\n    grid_eps=0.02\n)\n\n# Move the model to the appropriate device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:38:37.392265Z","iopub.execute_input":"2024-08-17T14:38:37.392658Z","iopub.status.idle":"2024-08-17T14:38:37.91291Z","shell.execute_reply.started":"2024-08-17T14:38:37.392617Z","shell.execute_reply":"2024-08-17T14:38:37.911843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Initialization of network","metadata":{}},{"cell_type":"markdown","source":"Input Layer is 784 i.e. 28 x 28 which is dimension of image <br>\nHidden layer is 2 x input_layer+1 i.e. 1569 <br>\nOutput layer is 10 for all digits","metadata":{}},{"cell_type":"code","source":"# Define model\nmodel = KAN([28 * 28, 28 * 28 * 2 + 1, 10])\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:38:38.772428Z","iopub.execute_input":"2024-06-13T08:38:38.772947Z","iopub.status.idle":"2024-06-13T08:38:39.440769Z","shell.execute_reply.started":"2024-06-13T08:38:38.772917Z","shell.execute_reply":"2024-06-13T08:38:39.439918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defining Optimizer and Learning Rate Decay","metadata":{}},{"cell_type":"code","source":"# Define optimizer\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n# Define learning rate scheduler\nscheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:38:41.700537Z","iopub.execute_input":"2024-08-17T14:38:41.701356Z","iopub.status.idle":"2024-08-17T14:38:41.706654Z","shell.execute_reply.started":"2024-08-17T14:38:41.701327Z","shell.execute_reply":"2024-08-17T14:38:41.705817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define loss\ncriterion = nn.CrossEntropyLoss()\n\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:38:46.397152Z","iopub.execute_input":"2024-08-17T14:38:46.397503Z","iopub.status.idle":"2024-08-17T14:38:46.402707Z","shell.execute_reply.started":"2024-08-17T14:38:46.397473Z","shell.execute_reply":"2024-08-17T14:38:46.401701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training of Network","metadata":{}},{"cell_type":"code","source":"for epoch in range(20):\n    # Train\n    model.train()\n    epoch_train_loss = 0\n    epoch_train_accuracy = 0\n    with tqdm(trainloader) as pbar:\n        for i, (images, labels) in enumerate(pbar):\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n            accuracy = (output.argmax(dim=1) == labels).float().mean()\n            epoch_train_loss += loss.item()\n            epoch_train_accuracy += accuracy.item()\n            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n    \n    epoch_train_loss /= len(trainloader)\n    epoch_train_accuracy /= len(trainloader)\n    train_losses.append(epoch_train_loss)\n    train_accuracies.append(epoch_train_accuracy)\n\n    # Validation\n    model.eval()\n    val_loss = 0\n    val_accuracy = 0\n    all_labels = []\n    all_preds = []\n    with torch.no_grad():\n        for images, labels in valloader:\n            images = images.view(-1, 28 * 28).to(device)\n            labels = labels.to(device)\n            output = model(images)\n            val_loss += criterion(output, labels).item()\n            val_accuracy += (output.argmax(dim=1) == labels).float().mean().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_preds.extend(output.argmax(dim=1).cpu().numpy())\n\n    val_loss /= len(valloader)\n    val_accuracy /= len(valloader)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_accuracy)\n\n    # Update learning rate\n    scheduler.step()\n\n    print(f\"Epoch {epoch + 1}, Train Loss: {epoch_train_loss}, Train Accuracy: {epoch_train_accuracy}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:38:49.846859Z","iopub.execute_input":"2024-08-17T14:38:49.847456Z","iopub.status.idle":"2024-08-17T14:41:51.716517Z","shell.execute_reply.started":"2024-08-17T14:38:49.847428Z","shell.execute_reply":"2024-08-17T14:41:51.715614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training Plot of Network","metadata":{}},{"cell_type":"code","source":"# Plot training and validation loss and accuracy\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Train Accuracy')\nplt.plot(val_accuracies, label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:41:33.439692Z","iopub.execute_input":"2024-06-13T08:41:33.440006Z","iopub.status.idle":"2024-06-13T08:41:33.893442Z","shell.execute_reply.started":"2024-06-13T08:41:33.439964Z","shell.execute_reply":"2024-06-13T08:41:33.892518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion Matrix Plot","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\n# Define class labels\nclass_labels = np.arange(10)\n\n# Confusion Matrix\ncm = confusion_matrix(all_labels, all_preds)\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_labels, yticklabels=class_labels)\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-13T08:41:33.895809Z","iopub.execute_input":"2024-06-13T08:41:33.896096Z","iopub.status.idle":"2024-06-13T08:41:34.427667Z","shell.execute_reply.started":"2024-06-13T08:41:33.896071Z","shell.execute_reply":"2024-06-13T08:41:34.426766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creation of submission file","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ntest_data = torch.tensor(test_df.values, dtype=torch.float32)\n\ntest_data = (test_data - 0.5) / 0.5\ntest_data = test_data.to(device)\nmodel.eval()\nwith torch.no_grad():\n    test_output = model(test_data)\n    test_predictions = test_output.argmax(dim=1).cpu().numpy()\n\nsubmission_df = pd.DataFrame({'ImageId': range(1, len(test_predictions) + 1), 'Label': test_predictions})\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-17T14:42:00.31073Z","iopub.execute_input":"2024-08-17T14:42:00.311445Z","iopub.status.idle":"2024-08-17T14:42:02.636382Z","shell.execute_reply.started":"2024-08-17T14:42:00.311415Z","shell.execute_reply":"2024-08-17T14:42:02.635602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}