{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook I tried to write NeuralNetwork from scratch using one hidden layer and some popular activation functions. You can use this notebook and make it better by adding new methods of updating weights or using batches. Have fun and good luck:)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom keras.datasets import mnist\nfrom keras.utils.np_utils import to_categorical\nimport numpy as np\nimport pandas as pd\nimport time\n\n(x_train, y_train), (x_val, y_val) = mnist.load_data()\nprint('Train: X=%s, y=%s' % (x_train.shape, y_train.shape))\nprint('Test: X=%s, y=%s' % (x_val.shape, y_val.shape))\n","metadata":{"execution":{"iopub.status.busy":"2021-10-30T15:58:38.007491Z","iopub.execute_input":"2021-10-30T15:58:38.008371Z","iopub.status.idle":"2021-10-30T15:58:38.336245Z","shell.execute_reply.started":"2021-10-30T15:58:38.008334Z","shell.execute_reply":"2021-10-30T15:58:38.335086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(9):\n    plt.subplot(330 + 1 + i)\n    plt.imshow(x_train[i], cmap=plt.get_cmap('gray'))","metadata":{"execution":{"iopub.status.busy":"2021-10-30T15:58:38.338060Z","iopub.execute_input":"2021-10-30T15:58:38.338366Z","iopub.status.idle":"2021-10-30T15:58:39.253978Z","shell.execute_reply.started":"2021-10-30T15:58:38.338334Z","shell.execute_reply":"2021-10-30T15:58:39.252955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.reshape((x_train.shape[0], 28*28)).astype('float64') / 255\nx_val = x_val.reshape((x_val.shape[0], 28*28)).astype('float64') / 255\ny_train = to_categorical(y_train)\ny_val = to_categorical(y_val)","metadata":{"execution":{"iopub.status.busy":"2021-10-30T15:58:39.255406Z","iopub.execute_input":"2021-10-30T15:58:39.255744Z","iopub.status.idle":"2021-10-30T15:58:39.432370Z","shell.execute_reply.started":"2021-10-30T15:58:39.255683Z","shell.execute_reply":"2021-10-30T15:58:39.431435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DeepNeuralNetwork():\n    def __init__(self, sizes, epochs=15, l_rate=0.005):\n        self.sizes = sizes\n        self.epochs = epochs\n        self.l_rate = l_rate\n        self.params = self.initialization()\n\n    def relu(self, x, derivative=False):\n        if derivative:\n            return [0 if i < 0 else 1 for i in x]\n        \n        return np.maximum(x, 0)\n\n    def softmax(self, x, derivative=False):\n        exps = np.exp(x - x.max())\n        if derivative:\n            return exps / np.sum(exps, axis=0) * (1 - exps / np.sum(exps, axis=0))\n        \n        return exps / np.sum(exps, axis=0)\n\n    def initialization(self):\n        input_layer=self.sizes[0]\n        hidden_layer=self.sizes[1]\n        output_layer=self.sizes[2]\n\n        params = {\n            'W1':np.random.randn(hidden_layer, input_layer) * np.sqrt(1. / hidden_layer),\n            'W2':np.random.randn(output_layer, hidden_layer) * np.sqrt(1. / output_layer)\n        }\n\n        return params\n\n    def forward_pass(self, x_train):\n        params = self.params\n\n        params['A0'] = x_train\n\n        params['Z1'] = np.dot(params[\"W1\"], params['A0'])\n        params['A1'] = self.relu(params['Z1'])\n\n        params['Z2'] = np.dot(params[\"W2\"], params['A1'])\n        params['A2'] = self.softmax(params['Z2'])\n\n        return params['A2']\n\n    def backward_pass(self, y_train, output):\n        \n        params = self.params\n        change_w = {}\n\n        error = 2 * (output - y_train) / output.shape[0] * self.softmax(params['Z2'], derivative=True)\n        change_w['W2'] = np.outer(error, params['A1'])\n\n        error = np.dot(params['W2'].T, error) * self.relu(params['Z1'], derivative=True)\n        change_w['W1'] = np.outer(error, params['A0'])\n\n        return change_w\n\n    def update_network_parameters(self, changes_to_w):\n        for key, value in changes_to_w.items():\n            self.params[key] -= self.l_rate * value\n\n    def cross_entropy(self, x_val, y_val):\n        predictions = []\n        for x in x_val:\n            predictions.append(self.forward_pass(x))\n        predictions = np.array(predictions)\n        return -np.mean(np.multiply(y_val,np.log(predictions)))\n\n    \n    def shuffle(self,x_train,y_train):\n        idx = [i for i in range(x_train.shape[0])]\n        np.random.shuffle(idx)\n        self.input = x_train[idx]\n        self.target = y_train[idx]\n    \n    def train(self, x_train, y_train, x_val, y_val):\n        start_time = time.time()\n        test_loss = []\n        iterat = range(self.epochs)\n        for iteration in range(self.epochs):\n            self.shuffle(x_train, y_train)\n            for x,y in zip(x_train, y_train):\n                output = self.forward_pass(x)\n    \n                changes_to_w = self.backward_pass(y, output)\n                self.update_network_parameters(changes_to_w)\n                \n    \n            cross_entropy_test = self.cross_entropy(x_val, y_val)\n            print('Epoch: {0}, Time Spent: {1:.2f}s, cross_entropy_test: {2:.4f}'.format(\n                iteration+1, time.time() - start_time, cross_entropy_test\n            ))\n            test_loss.append(cross_entropy_test)\n        plt.plot(iterat, test_loss, '-r')\n        plt.xlabel(\"epochs\")\n        plt.ylabel(\"test_loss\")","metadata":{"execution":{"iopub.status.busy":"2021-10-30T15:58:39.434700Z","iopub.execute_input":"2021-10-30T15:58:39.435050Z","iopub.status.idle":"2021-10-30T15:58:39.459901Z","shell.execute_reply.started":"2021-10-30T15:58:39.435006Z","shell.execute_reply":"2021-10-30T15:58:39.458968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dnn = DeepNeuralNetwork(sizes=[784, 128, 10])\ndnn.train(x_train, y_train, x_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2021-10-30T15:58:39.461086Z","iopub.execute_input":"2021-10-30T15:58:39.461315Z","iopub.status.idle":"2021-10-30T16:13:16.173102Z","shell.execute_reply.started":"2021-10-30T15:58:39.461288Z","shell.execute_reply":"2021-10-30T16:13:16.172144Z"},"trusted":true},"execution_count":null,"outputs":[]}]}