{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":221.410657,"end_time":"2024-06-13T08:45:32.484896","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-06-13T08:41:51.074239","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This is implementation of hermite Interpolated Network for Mnist dataset","metadata":{}},{"cell_type":"markdown","source":"## Please give upvote if you find this interesting 👍","metadata":{"papermill":{"duration":0.006671,"end_time":"2024-06-13T08:41:53.893266","exception":false,"start_time":"2024-06-13T08:41:53.886595","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Importing package","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-10T13:00:10.073448Z","iopub.execute_input":"2024-10-10T13:00:10.073954Z","iopub.status.idle":"2024-10-10T13:00:10.080259Z","shell.execute_reply.started":"2024-10-10T13:00:10.073910Z","shell.execute_reply":"2024-10-10T13:00:10.079130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Defining hermite Layers","metadata":{}},{"cell_type":"code","source":"class HermiteLayer(nn.Module):\n    def __init__(self, in_features, out_features, num_bases=5):\n        super(HermiteLayer, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.num_bases = num_bases\n\n        # Define Hermite weight (out_features x in_features * num_bases)\n        self.hermite_weight = nn.Parameter(torch.Tensor(out_features, in_features * num_bases))\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        nn.init.kaiming_uniform_(self.hermite_weight, a=0)\n\n    def hermite_polynomials(self, x):\n        \"\"\"\n        Generate Hermite polynomials up to the given order.\n        \"\"\"\n        batch_size, input_size = x.size()\n        assert input_size == self.in_features, \"Input size must match in_features\"\n        \n        # Generate Hermite basis functions for each input point\n        h0 = torch.ones_like(x)  # H0(x) = 1\n        h1 = 2 * x               # H1(x) = 2x\n        bases = [h0, h1]\n        \n        for n in range(2, self.num_bases):\n            h_next = 2 * x * bases[-1] - 2 * (n - 1) * bases[-2]\n            bases.append(h_next)\n\n        # Concatenate the Hermite basis functions along the feature axis\n        hermite_bases = torch.stack(bases, dim=-1)  # Shape: [batch_size, in_features, num_bases]\n        \n        return hermite_bases\n\n    def forward(self, x):\n        \"\"\"\n        Forward pass through Hermite interpolation.\n        \"\"\"\n        batch_size = x.size(0)\n\n        # Compute Hermite polynomials (output shape: [batch_size, in_features, num_bases])\n        hermite_bases = self.hermite_polynomials(x)\n\n        # Reshape the Hermite polynomials to prepare for matrix multiplication\n        hermite_bases = hermite_bases.view(batch_size, -1)  # Shape: [batch_size, in_features * num_bases]\n\n        # Linear transformation using the Hermite weight\n        hermite_output = F.linear(hermite_bases, self.hermite_weight)  # Shape: [batch_size, out_features]\n\n        return hermite_output\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-10T13:00:10.082243Z","iopub.execute_input":"2024-10-10T13:00:10.082714Z","iopub.status.idle":"2024-10-10T13:00:10.096612Z","shell.execute_reply.started":"2024-10-10T13:00:10.082663Z","shell.execute_reply":"2024-10-10T13:00:10.095547Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Defining Hermite Interpolation Network (HIN)","metadata":{}},{"cell_type":"code","source":"class HermiteInterpolationNetwork(nn.Module):\n    def __init__(self, in_features, out_features, num_bases=5):\n        super(HermiteInterpolationNetwork, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n\n        # Base linear layer weight\n        self.base_weight = nn.Parameter(torch.Tensor(out_features, in_features))\n        self.reset_parameters()\n\n        # Hermite interpolation layer\n        self.hermite_layer = HermiteLayer(in_features, out_features, num_bases)\n\n    def reset_parameters(self):\n        nn.init.kaiming_uniform_(self.base_weight, a=0)\n\n    def forward(self, x):\n        \"\"\"\n        Compute the base linear output and the Hermite output.\n        Combine them to form the final output.\n        \"\"\"\n        batch_size = x.size(0)\n\n        # Base linear transformation\n        base_output = F.linear(x, self.base_weight)  # Shape: [batch_size, out_features]\n\n        # Hermite interpolation transformation\n        hermite_output = self.hermite_layer(x)  # Shape: [batch_size, out_features]\n\n        # Combine base output and Hermite output\n        output = base_output + hermite_output  # Element-wise addition\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-10T13:00:10.097956Z","iopub.execute_input":"2024-10-10T13:00:10.098330Z","iopub.status.idle":"2024-10-10T13:00:10.113304Z","shell.execute_reply.started":"2024-10-10T13:00:10.098291Z","shell.execute_reply":"2024-10-10T13:00:10.112247Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Defining HIN for Mnist","metadata":{}},{"cell_type":"code","source":"class MNISTHermiteClassifier(nn.Module):\n    def __init__(self, input_size, hidden_size, output_size, num_bases=5):\n        super(MNISTHermiteClassifier, self).__init__()\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.output_size = output_size\n\n        # First layer: base + Hermite interpolation\n        self.layer1 = HermiteInterpolationNetwork(input_size, hidden_size, num_bases)\n        \n        # Second layer: standard linear layer\n        self.layer2 = nn.Linear(hidden_size, output_size)\n\n    def forward(self, x):\n        x = x.view(x.size(0), -1)  # Flatten input (batch_size, input_size)\n        x = self.layer1(x)         # First layer: Hermite interpolation\n        x = F.relu(x)              # ReLU activation\n        x = self.layer2(x)         # Second layer: linear output\n        return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-10T13:00:10.115800Z","iopub.execute_input":"2024-10-10T13:00:10.116312Z","iopub.status.idle":"2024-10-10T13:00:10.127719Z","shell.execute_reply.started":"2024-10-10T13:00:10.116260Z","shell.execute_reply":"2024-10-10T13:00:10.126480Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Defining Training Loop and plotting functions","metadata":{}},{"cell_type":"code","source":"def train_model(base):\n    # Hyperparameters\n    batch_size = 64\n    learning_rate = 0.0005\n    num_epochs = 20\n    validation_split = 0.2  # 20% for validation\n\n    # Load MNIST dataset\n    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])\n    train_dataset = datasets.MNIST(root='./data', train=True, transform=transform, download=True)\n    \n    # Split the dataset into training and validation sets\n    train_size = int(len(train_dataset) * (1 - validation_split))\n    val_size = len(train_dataset) - train_size\n    train_dataset, val_dataset = torch.utils.data.random_split(train_dataset, [train_size, val_size])\n\n    train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False)\n\n    # Define model, loss function, and optimizer\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = MNISTHermiteClassifier(input_size=28*28, hidden_size=256, output_size=10, num_bases=base).to(device)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\n    # Track loss and accuracy\n    train_losses = []\n    val_losses = []\n    train_accuracies = []\n    val_accuracies = []\n\n    # Training loop\n    for epoch in range(num_epochs):\n        model.train()  # Set model to training mode\n        running_loss = 0.0\n        correct = 0\n        total = 0\n\n        for batch_idx, (images, labels) in enumerate(train_loader):\n            images, labels = images.to(device), labels.to(device)\n\n            # Forward pass\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n\n            # Backward pass and optimization\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n            #if (batch_idx + 1) % 750 == 0:\n            #    print(f'Epoch [{epoch + 1}/{num_epochs}], Step [{batch_idx + 1}/{len(train_loader)}], Loss: {loss.item():.4f}')\n\n        # Calculate average loss and accuracy for the epoch\n        avg_loss = running_loss / len(train_loader)\n        accuracy = 100 * correct / total\n        train_losses.append(avg_loss)\n        train_accuracies.append(accuracy)\n\n        # Validation step\n        model.eval()  # Set model to evaluation mode\n        val_running_loss = 0.0\n        val_correct = 0\n        val_total = 0\n\n        with torch.no_grad():\n            for val_images, val_labels in val_loader:\n                val_images, val_labels = val_images.to(device), val_labels.to(device)\n                val_outputs = model(val_images)\n                val_loss = criterion(val_outputs, val_labels)\n\n                val_running_loss += val_loss.item()\n                _, val_predicted = torch.max(val_outputs.data, 1)\n                val_total += val_labels.size(0)\n                val_correct += (val_predicted == val_labels).sum().item()\n\n        # Calculate validation loss and accuracy\n        avg_val_loss = val_running_loss / len(val_loader)\n        val_accuracy = 100 * val_correct / val_total\n        val_losses.append(avg_val_loss)\n        val_accuracies.append(val_accuracy)\n\n        print(f'Epoch [{epoch + 1}/{num_epochs}], Loss: {avg_loss:.4f}, Accuracy: {accuracy:.2f}%, Val Loss: {avg_val_loss:.4f}, Val Accuracy: {val_accuracy:.2f}%')\n\n\n    plt.figure(figsize=(12, 5))\n\n    # Loss plot\n    plt.subplot(1, 2, 1)\n    plt.plot(train_losses, label='Training Loss', color='blue')\n    plt.plot(val_losses, label='Validation Loss', color='orange')\n    plt.title('Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    # Accuracy plot\n    plt.subplot(1, 2, 2)\n    plt.plot(train_accuracies, label='Training Accuracy', color='blue')\n    plt.plot(val_accuracies, label='Validation Accuracy', color='orange')\n    plt.title('Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy (%)')\n    plt.legend()\n\n    plt.tight_layout()\n    plt.show()\n\n    # Save the model if needed\n    torch.save(model.state_dict(), 'mnist_hermite_classifier.pth')\n\n    # Create submission CSV\n    test_dataset = datasets.MNIST(root='./data', train=False, transform=transform, download=True)\n    test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n\n    model.eval()  # Set model to evaluation mode\n    all_predictions = []\n    all_labels = []\n\n    with torch.no_grad():\n        for images, labels in test_loader:\n            images = images.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n\n    # Compute confusion matrix and accuracy for train, val, and test sets\n    def evaluate(loader, dataset_name):\n        model.eval()  # Set model to evaluation mode\n        all_predictions = []\n        all_labels = []\n\n        with torch.no_grad():\n            for images, labels in loader:\n                images = images.to(device)\n                outputs = model(images)\n                _, predicted = torch.max(outputs.data, 1)\n                all_predictions.extend(predicted.cpu().numpy())\n                all_labels.extend(labels.cpu().numpy())\n\n        conf_matrix = confusion_matrix(all_labels, all_predictions)\n        accuracy = accuracy_score(all_labels, all_predictions) * 100\n        #print(f'Confusion Matrix for {dataset_name}:\\n{conf_matrix}')\n        print(f'Accuracy for {dataset_name}: {accuracy:.2f}%')\n\n    # Evaluate on training and validation sets\n    evaluate(train_loader, \"Training\")\n    evaluate(val_loader, \"Validation\")\n    evaluate(test_loader, \"Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-10T13:02:52.265520Z","iopub.execute_input":"2024-10-10T13:02:52.266004Z","iopub.status.idle":"2024-10-10T13:02:52.298853Z","shell.execute_reply.started":"2024-10-10T13:02:52.265960Z","shell.execute_reply":"2024-10-10T13:02:52.297645Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Running Code and Training models on various Basis Functions","metadata":{}},{"cell_type":"code","source":"for i in range(2,8):\n    print(\"--------------------------------- Training model Basis: \"+ str(i) + \"---------------------------------\")\n    train_model(i)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-10-10T13:02:57.433199Z","iopub.execute_input":"2024-10-10T13:02:57.433639Z","iopub.status.idle":"2024-10-10T13:03:39.751006Z","shell.execute_reply.started":"2024-10-10T13:02:57.433595Z","shell.execute_reply":"2024-10-10T13:03:39.749413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}