{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import necessary libraries\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import DataLoader, TensorDataset\n\n# Load and preprocess the data\ndata = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nm, n = data.shape\nprint(m, n)\ndata_test_size = 1000/m\nprint(data_test_size)\n\n# Separate features and labels\nX = data.iloc[:, 1:].values  # pixel values\nY = data.iloc[:, 0].values   # labels (0-9)\n\n# Normalize pixel values to be between 0 and 1\nX = X / 255.0\n\n# Convert data to torch tensors\nX = torch.tensor(X, dtype=torch.float32)\nY = torch.tensor(Y, dtype=torch.long)\n\n# Split into training and development sets\nX_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size=data_test_size, random_state=42)\n\n# Create DataLoader for batching\ntrain_data = TensorDataset(X_train, Y_train)\ntrain_loader = DataLoader(train_data, batch_size=128, shuffle=True)\n\ndev_data = TensorDataset(X_dev, Y_dev)\ndev_loader = DataLoader(dev_data, batch_size=128, shuffle=False)","metadata":{"_uuid":"0942aaa6-ac4e-4810-a2df-901737fb667e","_cell_guid":"8ac3114a-815a-42a5-b6e9-87f6206ac69c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-16T13:04:23.968774Z","iopub.execute_input":"2024-10-16T13:04:23.969316Z","iopub.status.idle":"2024-10-16T13:04:32.296256Z","shell.execute_reply.started":"2024-10-16T13:04:23.969260Z","shell.execute_reply":"2024-10-16T13:04:32.295080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom initializers\ndef custom_initializer_1(layer):\n    if isinstance(layer, nn.Linear):\n        nn.init.uniform_(layer.weight, -0.5, 0.5)\n        nn.init.uniform_(layer.bias, -0.5, 0.5)\n\ndef custom_initializer_2(layer):\n    if isinstance(layer, nn.Linear):\n        nn.init.normal_(layer.weight, mean=0.0, std=4/(784+10))\n        nn.init.zeros_(layer.bias)\n\n# New model architecture\nclass SimpleNN(nn.Module):\n    def __init__(self, initializer=None):\n        super(SimpleNN, self).__init__()\n        self.fc1 = nn.Linear(784, 10)\n        self.fc2 = nn.Linear(10, 10)\n\n        # Apply the custom initializer if provided\n        if initializer is not None:\n            initializer(self.fc1)\n            initializer(self.fc2)\n\n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x","metadata":{"_uuid":"6e7f58d5-10b0-4fcb-a9d3-30d4373d4557","_cell_guid":"ec597882-2fe9-4e49-96cf-c5da2b11537d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-16T13:04:32.298762Z","iopub.execute_input":"2024-10-16T13:04:32.299501Z","iopub.status.idle":"2024-10-16T13:04:32.309560Z","shell.execute_reply.started":"2024-10-16T13:04:32.299456Z","shell.execute_reply":"2024-10-16T13:04:32.308422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop\ndef train_model(model, train_loader, dev_loader, epochs=50, learning_rate=0.1):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n\n    train_acc_history = []\n    dev_acc_history = []\n\n    for epoch in range(epochs):\n        model.train()\n        correct, total = 0, 0\n        running_loss = 0.0\n\n        for batch_x, batch_y in train_loader:\n            optimizer.zero_grad()\n            outputs = model(batch_x)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            optimizer.step()\n\n            # Track accuracy\n            _, predicted = torch.max(outputs, 1)\n            total += batch_y.size(0)\n            correct += (predicted == batch_y).sum().item()\n            running_loss += loss.item()\n\n        train_acc = 100 * correct / total\n        train_acc_history.append(train_acc)\n\n        # Evaluate on development set\n        model.eval()\n        dev_correct, dev_total = 0, 0\n        with torch.no_grad():\n            for dev_x, dev_y in dev_loader:\n                dev_outputs = model(dev_x)\n                _, dev_predicted = torch.max(dev_outputs, 1)\n                dev_total += dev_y.size(0)\n                dev_correct += (dev_predicted == dev_y).sum().item()\n\n        dev_acc = 100 * dev_correct / dev_total\n        dev_acc_history.append(dev_acc)\n\n        print(f'Epoch {epoch + 1}/{epochs}, Loss: {running_loss / len(train_loader):.4f}, Train Accuracy: {train_acc:.2f}%, Dev Accuracy: {dev_acc:.2f}%')\n\n    return train_acc_history, dev_acc_history","metadata":{"_uuid":"c5b490b5-4f08-41f2-b947-2b98a289e0d1","_cell_guid":"ced56fe2-40c1-4e36-951d-45c3cbfb881f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-16T13:04:32.311100Z","iopub.execute_input":"2024-10-16T13:04:32.311481Z","iopub.status.idle":"2024-10-16T13:04:32.326012Z","shell.execute_reply.started":"2024-10-16T13:04:32.311442Z","shell.execute_reply":"2024-10-16T13:04:32.324679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting function for accuracy\ndef plot_history(train_accs, dev_accs, names):\n    for i, name in enumerate(names):\n        plt.plot(train_accs[i], label=f'{name} Train Accuracy')\n        plt.plot(dev_accs[i], label=f'{name} Dev Accuracy')\n    plt.title('Model Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()\n\n# Train and evaluate with both custom initializations and default PyTorch initialization\ninitializers = {\n    'Initializer 1 (Uniform)': custom_initializer_1,\n    'Initializer 2 (Normal)': custom_initializer_2,\n    'Default (PyTorch Init)': None\n}\n\ntrain_accs = []\ndev_accs = []\n\nfor name, initializer in initializers.items():\n    print(f\"Training model with {name}\")\n    \n    # Build model with the current initializer\n    model = SimpleNN(initializer)\n    \n    # Train the model\n    train_acc, dev_acc = train_model(model, train_loader, dev_loader)\n    \n    # Store results for plotting\n    train_accs.append(train_acc)\n    dev_accs.append(dev_acc)\n\n# Plot all accuracies\nplot_history(train_accs, dev_accs, initializers.keys())\n\n# Predict a sample and visualize\nindex = 5  # Pick a sample index\ncurrent_image = X_dev[index].reshape(28, 28).numpy()  # Reshape from 784 to 28x28\n\n# Display the image\nplt.imshow(current_image, cmap='gray')\nplt.axis('off')  # Hide axis\nplt.show()\n\n# Make a prediction with the last trained model (using last initializer)\nmodel.eval()\nsample = X_dev[index].reshape(1, 784)\nprediction = torch.argmax(model(sample)).item()\ntrue_label = Y_dev[index].item()\n\nprint(f\"Prediction: {prediction}, True Label: {true_label}\")","metadata":{"_uuid":"a8f33ce3-62d2-4efb-9c28-22700369f5ea","_cell_guid":"2065f54e-946a-49fb-8924-ff688762c1a0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-10-16T13:04:32.328616Z","iopub.execute_input":"2024-10-16T13:04:32.328996Z","iopub.status.idle":"2024-10-16T13:06:37.518429Z","shell.execute_reply.started":"2024-10-16T13:04:32.328958Z","shell.execute_reply":"2024-10-16T13:06:37.516458Z"},"trusted":true},"execution_count":null,"outputs":[]}]}