{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"IMPORTING PACKAGES BELOW-","metadata":{}},{"cell_type":"code","source":"# creating a neural network for mnist image classification in the DigitRecognizer competition on kaggle\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom sklearn.model_selection import train_test_split\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-11T14:34:55.424960Z","iopub.execute_input":"2024-08-11T14:34:55.425422Z","iopub.status.idle":"2024-08-11T14:34:58.077545Z","shell.execute_reply.started":"2024-08-11T14:34:55.425384Z","shell.execute_reply":"2024-08-11T14:34:58.076434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Linear Layer Class:\n\nImplements a fully connected layer.\n\nMethods: forward, backward.","metadata":{}},{"cell_type":"code","source":"class Layer:\n    def forward(self, inputs):\n        \"\"\"\n        Compute the forward pass.\n        \"\"\"\n        raise NotImplementedError\n\n    def backward(self, gradient):\n        \"\"\"\n        Compute the backward pass.\n        \"\"\"\n        raise NotImplementedError\n\nclass Linear(Layer):\n    \"\"\"Fully connected layer.\"\"\"\n\n    def __init__(self, input_size, output_size):\n        \"\"\"\n        Initialize the layer.\n        \"\"\"\n        self.weights = np.random.randn(input_size, output_size) * 0.01\n        self.biases = np.zeros((1, output_size))\n        self.inputs = None\n        self.gradients = {'weights': None, 'biases': None}\n\n    def forward(self, inputs):\n        \"\"\"Compute forward pass.\"\"\"\n        self.inputs = inputs\n        return np.dot(inputs, self.weights) + self.biases\n\n    def backward(self, gradient):\n        \"\"\"Compute backward pass.\"\"\"\n        self.gradients['weights'] = np.dot(self.inputs.T, gradient)\n        self.gradients['biases'] = np.sum(gradient, axis=0, keepdims=True)\n        return np.dot(gradient, self.weights.T)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.080142Z","iopub.execute_input":"2024-08-11T14:34:58.080668Z","iopub.status.idle":"2024-08-11T14:34:58.093240Z","shell.execute_reply.started":"2024-08-11T14:34:58.080636Z","shell.execute_reply":"2024-08-11T14:34:58.091789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ReLU Activation Class:\n\nImplements the ReLU activation function.\n\nMethods: forward, backward.\n","metadata":{}},{"cell_type":"code","source":"class ReLU(Layer):\n    \"\"\"ReLU activation layer.\"\"\"\n    \n    def __init__(self):\n        self.inputs = None\n        \n    def forward(self, inputs):\n        \"\"\"Apply ReLU activation.\"\"\"\n        self.inputs = inputs\n        return np.maximum(0, inputs)\n\n    def backward(self, gradient):\n        \"\"\"Compute gradient of ReLU.\"\"\"\n        return gradient * (self.inputs > 0)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.095101Z","iopub.execute_input":"2024-08-11T14:34:58.096117Z","iopub.status.idle":"2024-08-11T14:34:58.114101Z","shell.execute_reply.started":"2024-08-11T14:34:58.096070Z","shell.execute_reply":"2024-08-11T14:34:58.112744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sigmoid Activation Class:\n\nImplements the Sigmoid activation function.\n\nMethods: forward, backward.","metadata":{}},{"cell_type":"code","source":"class Sigmoid:\n    def __init__(self):\n        self.output = None\n\n    def forward(self, inputs):\n        \"\"\"\n        Perform the forward pass of the sigmoid function.\n        \n        :param inputs: Input values (can be a scalar or an array)\n        :return: Output of the sigmoid function\n        \"\"\"\n        self.output = 1 / (1 + np.exp(-inputs))\n        return self.output\n\n    def backward(self, values):\n        \"\"\"\n        Perform the backward pass of the sigmoid function.\n        \n        :param dvalues: Gradient of the loss with respect to the output of the sigmoid\n        :return: Gradient of the loss with respect to the input of the sigmoid\n        \"\"\"\n        return values * self.output * (1 - self.output)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.116071Z","iopub.execute_input":"2024-08-11T14:34:58.116648Z","iopub.status.idle":"2024-08-11T14:34:58.134057Z","shell.execute_reply.started":"2024-08-11T14:34:58.116607Z","shell.execute_reply":"2024-08-11T14:34:58.132692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tanh Activation Class:\n\nImplements the Tanh activation function.\n\nMethods: forward, backward.","metadata":{}},{"cell_type":"code","source":"class Tanh:\n    def __init__(self):\n        self.output = None\n\n    def forward(self, inputs):\n        \"\"\"\n        Perform the forward pass of the tanh function.\n        \n        :param inputs: Input values (can be a scalar or an array)\n        :return: Output of the tanh function\n        \"\"\"\n        self.output = np.tanh(inputs)\n        return self.output\n\n    def backward(self, dvalues):\n        \"\"\"\n        Perform the backward pass of the tanh function.\n        \n        :param dvalues: Gradient of the loss with respect to the output of tanh\n        :return: Gradient of the loss with respect to the input of tanh\n        \"\"\"\n        return dvalues * (1 - np.square(self.output))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.137610Z","iopub.execute_input":"2024-08-11T14:34:58.138031Z","iopub.status.idle":"2024-08-11T14:34:58.156445Z","shell.execute_reply.started":"2024-08-11T14:34:58.137997Z","shell.execute_reply":"2024-08-11T14:34:58.154614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Softmax Activation Class:\n\nImplements the Softmax activation function.\n\nMethods: forward, backward.","metadata":{}},{"cell_type":"code","source":"class Softmax(Layer):\n    \"\"\"Softmax activation layer.\"\"\"\n\n    def forward(self, inputs):\n        \"\"\"Apply Softmax activation.\"\"\"\n        exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))\n        self.probabilities = exp_values / np.sum(exp_values, axis=1, keepdims=True)\n        return self.probabilities\n\n    def backward(self, gradient):\n        \"\"\"Compute gradient of Softmax.\"\"\"\n        return gradient  # Gradient calculation is handled by CrossEntropyLoss","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.163391Z","iopub.execute_input":"2024-08-11T14:34:58.163900Z","iopub.status.idle":"2024-08-11T14:34:58.178922Z","shell.execute_reply.started":"2024-08-11T14:34:58.163859Z","shell.execute_reply":"2024-08-11T14:34:58.177049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Cross-Entropy Loss Class:\n\n   Implements the cross-entropy loss function. You can use the fusion method described in the PDF as    well. See how nn.CrossEntropyLoss in PyTorch works.\n\n   Methods: forward, backward.\n\n2. Mean Squared Error (MSE) Loss Class:\n\n   Implements the MSE loss function.\n\n   Methods: forward, backward\n","metadata":{}},{"cell_type":"code","source":"class Loss:\n    \"\"\"Base class for loss functions.\"\"\"\n\n    def forward(self, predictions, targets):\n        \"\"\"Compute the loss.\"\"\"\n        raise NotImplementedError\n\n    def backward(self, predictions, targets):\n        \"\"\"Compute the gradient of the loss.\"\"\"\n        raise NotImplementedError\n        \nclass CrossEntropyLoss(Loss):\n    \"\"\"Cross-entropy loss function.\"\"\"\n\n    def forward(self, predictions, targets):\n        \"\"\"Compute cross-entropy loss.\"\"\"\n        eps = 1e-15  # To avoid log(0) errors\n        predictions = np.clip(predictions, eps, 1 - eps)\n        return -np.sum(targets * np.log(predictions)) / predictions.shape[0]\n\n    def backward(self, predictions, targets):\n        \"\"\"Compute gradient of cross-entropy loss.\"\"\"\n        eps = 1e-15\n        predictions = np.clip(predictions, eps, 1 - eps)\n        return (predictions - targets) / predictions.shape[0]\n    \nclass MSELoss:\n    def __init__(self):\n        self.dinputs = None\n        self.y_pred = None\n        self.y_true = None\n\n    def forward(self, y_pred, y_true):\n        \"\"\"\n        Compute the Mean Squared Error loss.\n\n        :param y_pred: Predicted values\n        :param y_true: True values\n        :return: MSE loss\n        \"\"\"\n        self.y_pred = y_pred\n        self.y_true = y_true\n        \n        # Calculate loss\n        sample_losses = np.mean((y_true - y_pred)**2, axis=-1)\n        return np.mean(sample_losses)\n\n    def backward(self, dvalues=1):\n        \"\"\"\n        Compute the gradient of the loss with respect to the inputs.\n\n        :param dvalues: Gradient of the loss with respect to the output of this layer.\n                        Usually 1 unless we're using this as part of a larger network.\n        :return: Gradient of the loss with respect to the inputs (y_pred)\n        \"\"\"\n        # Number of samples\n        samples = len(self.y_pred)\n        # Number of outputs in every sample\n        outputs = len(self.y_pred[0])\n        \n        # Gradient on predictions\n        self.dinputs = -2 * (self.y_true - self.y_pred) / outputs\n        # Normalize gradient\n        self.dinputs = self.dinputs / samples\n        \n        return self.dinputs","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.180660Z","iopub.execute_input":"2024-08-11T14:34:58.181120Z","iopub.status.idle":"2024-08-11T14:34:58.217764Z","shell.execute_reply.started":"2024-08-11T14:34:58.181081Z","shell.execute_reply":"2024-08-11T14:34:58.215018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SGD Optimizer Class:\n\nImplements the stochastic gradient descent optimizer.\n\nMethods: step.\n","metadata":{}},{"cell_type":"code","source":"class SGD:\n    \"\"\"Stochastic Gradient Descent optimizer.\"\"\"\n\n    def __init__(self, learning_rate):\n        \"\"\"\n        Initialize the optimizer.\n        \"\"\"\n        self.learning_rate = learning_rate\n\n    def step(self, layer):\n        \"\"\"\n        Perform a single optimization step.\n        \"\"\"\n        layer.weights -= self.learning_rate * layer.gradients['weights']\n        layer.biases -= self.learning_rate * layer.gradients['biases']","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.220409Z","iopub.execute_input":"2024-08-11T14:34:58.221379Z","iopub.status.idle":"2024-08-11T14:34:58.228412Z","shell.execute_reply.started":"2024-08-11T14:34:58.221341Z","shell.execute_reply":"2024-08-11T14:34:58.227103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Model Class:\n\nWraps everything into a cohesive model.\n\nMethods: add_layer, compile, train, predict, evaluate, save, and load.","metadata":{}},{"cell_type":"code","source":"class Model:\n    \"\"\"Neural network model.\"\"\"\n\n    def __init__(self):\n        \"\"\"Initialize the model.\"\"\"\n        self.layers = []\n        self.loss = None\n        self.optimizer = None\n\n    def add_layer(self, layer):\n        \"\"\"Add a layer to the model.\"\"\"\n        self.layers.append(layer)\n\n    def compile(self, loss, optimizer):\n        \"\"\"Compile the model.\"\"\"\n        self.loss = loss\n        self.optimizer = optimizer\n\n    def forward(self, inputs):\n        \"\"\"Perform forward pass through all layers.\"\"\"\n        for layer in self.layers:\n            inputs = layer.forward(inputs)\n        return inputs\n\n    def backward(self, gradient):\n        \"\"\"Perform backward pass through all layers.\"\"\"\n        for layer in reversed(self.layers):\n            gradient = layer.backward(gradient)\n\n    def train(self, X, y, epochs, batch_size):\n        \"\"\"Train the model.\"\"\"\n        for epoch in range(epochs):\n            epoch_loss = 0\n            indices = np.arange(X.shape[0])\n            np.random.shuffle(indices)\n            X = X[indices]\n            y = y[indices]\n\n            for i in range(0, len(X), batch_size):\n                X_batch = X[i:i+batch_size]\n                y_batch = y[i:i+batch_size]\n\n                predictions = self.forward(X_batch)\n                batch_loss = self.loss.forward(predictions, y_batch)\n                epoch_loss += batch_loss\n\n                gradient = self.loss.backward(predictions, y_batch)\n                self.backward(gradient)\n\n                for layer in self.layers:\n                    if hasattr(layer, 'weights') and hasattr(layer, 'biases'):\n                        self.optimizer.step(layer)\n\n            print(f\"Epoch {epoch+1}/{epochs}, Loss: {epoch_loss/len(X):.4f}\")\n\n    def predict(self, X):\n        \"\"\"Make predictions on new data.\"\"\"\n        return self.forward(X)\n\n    def evaluate(self, X, y):\n        \"\"\"Evaluate the model.\"\"\"\n        predictions = self.predict(X)\n        loss = self.loss.forward(predictions, y)\n        accuracy = np.mean(np.argmax(predictions, axis=1) == np.argmax(y, axis=1))\n        return loss, accuracy\n    \n    def save(self, filepath):\n        \"\"\"Save the model weights to a file.\"\"\"\n        weights = []\n        for layer in self.layers:\n            if hasattr(layer, 'weights') and hasattr(layer, 'biases'):\n                weights.append((layer.weights, layer.biases))\n        with open(filepath, 'wb') as f:\n            pickle.dump(weights, f)\n        print(f\"Weights saved to {filepath}\")\n\n    def load(self, filepath):\n        \"\"\"Load the model weights from a file.\"\"\"\n        with open(filepath, 'rb') as f:\n            weights = pickle.load(f)\n    \n        weight_index = 0  # Counter for the weights list\n    \n        for layer in self.layers:\n            if hasattr(layer, 'weights') and hasattr(layer, 'biases'):\n                try:\n                    layer.weights, layer.biases = weights[weight_index]\n                    weight_index += 1  # Increment the counter only if weights were loaded\n                except IndexError:\n                    print(f\"Error: Mismatch in the number of layers. Could not load weights for layer {layer}.\")\n                    break\n        print(f\"Weights loaded from {filepath}\")","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.230145Z","iopub.execute_input":"2024-08-11T14:34:58.230882Z","iopub.status.idle":"2024-08-11T14:34:58.278399Z","shell.execute_reply.started":"2024-08-11T14:34:58.230836Z","shell.execute_reply":"2024-08-11T14:34:58.276888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One hot encoding - ","metadata":{}},{"cell_type":"code","source":"def one_hot_encode(y, num_classes):\n    return np.eye(num_classes)[y.astype(int).reshape(-1)]","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.279973Z","iopub.execute_input":"2024-08-11T14:34:58.280898Z","iopub.status.idle":"2024-08-11T14:34:58.294961Z","shell.execute_reply.started":"2024-08-11T14:34:58.280853Z","shell.execute_reply":"2024-08-11T14:34:58.293744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading and preprocessing data - \n","metadata":{}},{"cell_type":"code","source":"training_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nX_train = training_data.iloc[:, 1:].values\ny_train = training_data.iloc[:, 0].values\n\nX_train = X_train.astype('float32') / 255\ny_train = y_train.astype('int')\n\n# One-hot encode the labels\ny_train = one_hot_encode(y_train, 10)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:34:58.296588Z","iopub.execute_input":"2024-08-11T14:34:58.297601Z","iopub.status.idle":"2024-08-11T14:35:02.797421Z","shell.execute_reply.started":"2024-08-11T14:34:58.297553Z","shell.execute_reply":"2024-08-11T14:35:02.796148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Initializing, compiling and training the model - \n","metadata":{}},{"cell_type":"code","source":"model = Model()\nmodel.add_layer(Linear(784, 128))\nmodel.add_layer(ReLU())\nmodel.add_layer(Linear(128, 10))\nmodel.add_layer(Softmax())\n\nloss = CrossEntropyLoss()\noptimizer = SGD(learning_rate=0.01)\nmodel.compile(loss, optimizer)\n\n# Train the model\nmodel.train(X_train, y_train, epochs=20, batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:35:02.799300Z","iopub.execute_input":"2024-08-11T14:35:02.799794Z","iopub.status.idle":"2024-08-11T14:35:37.016086Z","shell.execute_reply.started":"2024-08-11T14:35:02.799752Z","shell.execute_reply":"2024-08-11T14:35:37.014564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now , it's time for the Test dataset -","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nX_test = np.array(test_data)\nX_test = X_test.astype('float32') / 255\ny_predicted = model.predict(X_test)\ny_predicted = np.argmax(y_predicted, axis=1)\nprint(y_predicted.T.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:35:37.018150Z","iopub.execute_input":"2024-08-11T14:35:37.019037Z","iopub.status.idle":"2024-08-11T14:35:39.925112Z","shell.execute_reply.started":"2024-08-11T14:35:37.018983Z","shell.execute_reply":"2024-08-11T14:35:39.920594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating a csv file with the Image ID and the label for submission in the competition -  ","metadata":{}},{"cell_type":"code","source":"import csv\nwith open('predictions.csv', 'w') as fileObj:\n    writerObj = csv.writer(fileObj)\n    writerObj.writerow(['ImageId','Label'])\n    for i in range(1,28001):\n        writerObj.writerow([i, y_predicted[i-1]])","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:35:39.931994Z","iopub.execute_input":"2024-08-11T14:35:39.932790Z","iopub.status.idle":"2024-08-11T14:35:40.029382Z","shell.execute_reply.started":"2024-08-11T14:35:39.932730Z","shell.execute_reply":"2024-08-11T14:35:40.028036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving the model - ","metadata":{}},{"cell_type":"code","source":"model.save('/kaggle/working/model_weights')","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:35:40.031255Z","iopub.execute_input":"2024-08-11T14:35:40.031714Z","iopub.status.idle":"2024-08-11T14:35:40.038764Z","shell.execute_reply.started":"2024-08-11T14:35:40.031675Z","shell.execute_reply":"2024-08-11T14:35:40.037607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading the weights and biases - ","metadata":{}},{"cell_type":"code","source":"model.load('/kaggle/working/model_weights')","metadata":{"execution":{"iopub.status.busy":"2024-08-11T14:35:40.040538Z","iopub.execute_input":"2024-08-11T14:35:40.040990Z","iopub.status.idle":"2024-08-11T14:35:40.050542Z","shell.execute_reply.started":"2024-08-11T14:35:40.040949Z","shell.execute_reply":"2024-08-11T14:35:40.049277Z"},"trusted":true},"execution_count":null,"outputs":[]}]}