{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Simple NN from Scratch MNIST\n\n* **1. Loading the data**\n    * 1.1 Training data\n    * 1.2 Test data\n    * 1.3 Normalize the data\n    * 1.4 Create batches\n* **2. Neural Network**\n    * 2.1 Create layer and activation objects\n    * 2.2 Create the network\n    * 2.3 Define backpropagation function and the training loop\n* **3. Training**\n    * 3.1 Start training\n* **4. Evaluation**\n    * 4.1 Test the model    \n    * 4.2 Make a submission    \n\nThis is a 3 layer neural network with a single dense layer. It uses backpropagation and gradient descent to fit to the data. **Without NN frameworks!** ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport sys, pickle\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading the data\n### 1.1 Training data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n\ntrain_y = list(df_train[\"label\"])\ntrain_X = np.array( df_train.drop(\"label\",axis = 1))\n\nsns.countplot(train_y)\nplt.show()\nplt.imshow(train_X[0].reshape((28,28)),cmap = \"gray_r\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.2 Test data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_X = np.array(pd.read_csv(\"../input/digit-recognizer/test.csv\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.3 Normalize the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalizing is important because it prevents gradient explosions\n\ntrain_X = train_X / 255.0\n\ntest_X = test_X / 255.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.4 Create batches","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_batches(X,y,batch_size = 16):\n    output_X = []\n    output_y = []\n    \n    \n    num_batches = int(len(X) / batch_size)\n    num_mod = np.mod(len(X),batch_size)\n    x = 0\n    for i in range(num_batches):\n        output_X.append(X[x:x+batch_size])\n        output_y.append(y[x:x+batch_size])\n        x = x + batch_size\n    if num_mod != 0:\n        if num_mod == 1:\n            output_X.append(X[-1])\n            output_y.append(y[-1])\n        else:\n            output_X.append([X[-(num_mod):-1][0],X[-1]])\n            output_y.append([y[-(num_mod):-1][0],y[-1]])\n        \n    return output_X,output_y\n\ntrain_X,train_y = create_batches(train_X,train_y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.1 Create layer objects","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Layer\nclass layer_dense:\n    def __init__(self,n_inputs,n_neurons):\n        #randomly initialize the weights an biases\n        self.weights = np.random.randn(n_inputs,n_neurons) * 0.1 # initialize it with the random numbers between 0 and 1\n        self.biases = np.zeros((1,n_neurons)) # initialize it with 0\n        self.name = \"layer_dense\"\n    def forward(self,inputs):\n        self.input = inputs # Save the input for the backpropagation algo\n        self.output = np.dot(inputs,self.weights) + self.biases # a * w + b\n        return self.output\n# Activations\nclass ReLU:\n    def __init__(self):\n        self.name = \"relu\"\n    def forward(self,inputs):\n        self.input = inputs\n        self.output = np.maximum(0,inputs) # Just looking if the number is bigger than zero\n        return self.output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.2 Create the nertwork","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# seed it\nnp.random.seed(10)\n\n# You can simply just add things to the network by adding objects to this list\nnet = [\n    layer_dense(784,200),\n    ReLU(),\n    layer_dense(200,10),\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.3 Define backpropagation function and the training loop","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyperparameters\nreg = 1e-3\nstep_size = 1e-3 # learning rate\n\nnum_samples = len(train_X[0])\n\n# Runs the network\ndef forward(inputs,net):\n    out = inputs\n    for layer in net:\n        out = layer.forward(out)\n    return out\n\ndef softmax(output):\n    # Get the probabilities with softmax\n    exp = np.exp(output)\n    probs = exp / np.sum(exp, axis=1, keepdims=True)\n    return probs\n\ndef backpropagation(net,dscores):\n    n = 1\n    for layer in reversed(net): # The loop starts at the end of the network\n        if layer.name == \"relu\":\n            # backpropagate the ReLU non-linearity\n            dscores[layer.output <= 0] = 0\n        else:\n            dW = 0\n            dB = 0\n            \n            # backpropate the gradient to the parameters\n            dW = np.dot(layer.input.T, dscores)\n            db = np.sum(dscores, axis=0, keepdims=True)\n            \n            # Check if it is the last layer \n            # if it isn't it will backpropagate into the next layer\n            if n != len(net):\n                dscores = np.dot(dscores, layer.weights.T)\n                \n            #Apply regularization\n            dW += reg*layer.weights\n            \n            #Apply gradients\n            layer.weights += -step_size * dW\n            layer.biases += -step_size * db\n        n += 1\n        \ndef train(epochs):\n    for i in range(epochs):\n        for i in range(len(train_X)-1):\n            \n            # Get the current batch\n            feat = train_X[i]\n            target = train_y[i]\n            \n            # Run batch trough the network\n            output = forward(feat,net)\n            \n            # Get the probabilities with softmax\n            probs = softmax(output)\n            \n            # Compute the gradient of the output\n            dscores = probs\n            dscores[range(num_samples),target] -= 1\n            dscores /= num_samples\n            dscores[output <= 0] = 0\n\n            backpropagation(net,dscores)\n            \n            sys.stdout.write(f\"\\rTraining - {round(i / len(train_X) * 100)}%\")\n            sys.stdout.flush()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3.1 Start training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can train for more epochs if you want\nEPOCHS = 3000\n\ntrain(EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.1 Test the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict on test data\nout = forward(test_X,net)\n# Apply sotmax and argmax\nsub = softmax(out).argmax(axis = 1)\nsns.countplot(sub)\nplt.show()\n# Show the distribution of the prediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.2 Make a submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\n\nsub_df[\"Label\"] = sub \n\nsub_df.to_csv(\"submission.csv\",index = False)\n\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}