{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "ab5f59ff-64a2-096e-a658-a36cd24c9385" }, "source": [ "This kernel presents a simple implementation of a 2-layer neural network. It heavily builds on the Machine Learning Mooc from Coursera.\n", "I know it's possible to do much simpler and faster with sklearn, but I want to post my own implementation as I first wanted to understand the algorithm before using black boxes.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f5bcdd73-036e-3493-d797-f8e5752dd526" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as nm # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt\n", "from scipy import optimize as opti\n", "\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "%matplotlib inline\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1c378b66-257d-834b-5bbf-14bc862722d0" }, "outputs": [], "source": [ "## Defining some functions.\n", "\n", "## This is the sigmoid activation function\n", "def sigmoid(z):\n", " return 1./(1+ nm.exp(-z));\n", "\n", "## This is the gradient of the sigmoid function. Used in the backprop\n", "def sigmoidGradient(z):\n", " return sigmoid(z)*(1-sigmoid(z));\n", "\n", " \n", "## This is the cost function, that return the cost and the gradient of the cost\n", "## as a function of the parameters nn_theta of the neural net. This function id hardcoded\n", "## for two hidden layers\n", "\n", "def nnCostFunction2(nn_params, input_layer_size, hidden_layer_size1,hidden_layer_size2, num_labels, X, y, lam):\n", "\n", " ## defining the number of parameters in each layer\n", " N_elem_theta1=hidden_layer_size1*(1+input_layer_size)\n", " N_elem_theta2=(hidden_layer_size1+1) *hidden_layer_size2\n", " N_elem_theta3=(hidden_layer_size2+1)*num_labels\n", "\n", "\n", " Theta1=nn_params[0:N_elem_theta1].reshape((hidden_layer_size1, input_layer_size+1))\n", " Theta2=nn_params[N_elem_theta1:N_elem_theta1+N_elem_theta2].reshape((hidden_layer_size2, hidden_layer_size1+1))\n", " Theta3=nn_params[N_elem_theta1+N_elem_theta2:].reshape((num_labels, hidden_layer_size2+1))\n", "\n", "\n", " m= X.shape[0]\n", " \n", " J = 0;\n", " Theta1_grad = nm.zeros(Theta1.shape);\n", " Theta2_grad = nm.zeros(Theta2.shape);\n", " Theta3_grad = nm.zeros(Theta3.shape);\n", "\n", " YY=nm.zeros((m, num_labels));\n", " for i in range(1, num_labels):\n", " YY[nm.where(y==i)[0], i-1]=1\n", " \n", " YY[nm.where(y==0), -1]=1\n", "\n", "\n", " a1=nm.append(nm.ones((m,1)), X, axis=1)\n", " z2=nm.dot(Theta1,a1.T)\n", " a2=sigmoid(z2).T;\n", " a2=nm.append(nm.ones((m,1)), a2, axis=1);\n", " z3=nm.dot(Theta2, a2.T)\n", " a3=sigmoid(z3).T;\n", " a3=nm.append(nm.ones((m,1)), a3, axis=1);\n", " z4=sigmoid(nm.dot(Theta3, a3.T));\n", "\n", " \n", " #J=nm.sum( -YY * nm.log(z3.T) - (1-YY)* nm.log(1-z3.T)) /m\n", " J=nm.sum( nm.log( (z4.T)**(-YY) * (1-z4.T)**(-(1-YY)) ))/m\n", " #print J, sum(YY), 1-nm.max(z3), nm.min(z3)\n", " ## adding regul\n", " J+=lam/(2.*m) *(nm.sum(Theta1[:,1:]**2) + nm.sum(Theta2[:,1:]**2)+nm.sum(Theta3[:,1:]**2) )\n", "\n", "\n", " #### using backprop to compute the gradient.\n", "\n", " Delta_1ij=nm.zeros(Theta1.shape);\n", " Delta_2ij=nm.zeros(Theta2.shape);\n", " Delta_3ij=nm.zeros(Theta3.shape);\n", "\n", "\n", " delta4 = z4.T-YY;\n", "\n", "\n", "# delta3 = z3.T-YY;\n", " \n", " ##3compute delta2\n", " delta3=(nm.dot(delta4, Theta3)[:, 1:] * sigmoidGradient(z3).T)\n", " delta2=(nm.dot(delta3, Theta2)[:, 1:] * sigmoidGradient(z2).T)\n", " ##4 accumulate\n", " \n", " \n", " \n", " Delta_1ij=nm.dot(delta2.T,a1)/m;\n", " Delta_2ij=nm.dot(delta3.T, a2)/m;\n", " Delta_3ij=nm.dot(delta4.T, a3)/m;\n", "\n", " Theta1_grad+=Delta_1ij;\n", " Theta2_grad+=Delta_2ij;\n", " Theta3_grad+=Delta_3ij;\n", " \n", " Theta1_grad[:,1:]+=lam/m * Theta1[:,1:];\n", " Theta2_grad[:,1:]+=lam/m * Theta2[:,1:];\n", " Theta3_grad[:,1:]+=lam/m * Theta3[:,1:];\n", " \n", " \n", " grad=nm.append(nm.append(Theta1_grad.flatten(), Theta2_grad.flatten()), Theta3_grad.flatten())\n", "\n", " return J, grad\n", "\n", "def predict2(Theta1, Theta2, Theta3,X):\n", "\n", " m = X.shape[0];\n", " num_labels = Theta3.shape[0];\n", " p = nm.zeros((X.shape[0], 1));\n", " X=nm.append(nm.ones((m,1)), X, axis=1)\n", " h1 = sigmoid(nm.dot(X , Theta1.T));\n", " h1= nm.append(nm.ones((m,1)), h1, axis=1)\n", " h2 = sigmoid(nm.dot(h1 , Theta2.T));\n", " h2=nm.append(nm.ones((m,1)), h2, axis=1)\n", " h3= sigmoid(nm.dot(h2 , Theta3.T));\n", " p=nm.argmax(h3, axis=1)+1\n", " p[p==10]=0\n", " return p\n", "\n", "def train2(X, y, hidden_layer_size1,hidden_layer_size2, lam):\n", " input_layer_size =X.shape[1]\n", " num_labels=10\n", "\n", " m=X.shape[0]\n", " print('there are {} training samples here'.format(m))\n", "\n", "\n", " eps=0.12\n", " ## initialing params.\n", " initial_Theta1 = nm.random.rand(hidden_layer_size1, 1+input_layer_size) *2*eps -eps \n", " initial_Theta2 = nm.random.rand(hidden_layer_size2, 1+ hidden_layer_size1) *2*eps -eps \n", " initial_Theta3 = nm.random.rand(num_labels, 1+ hidden_layer_size2) *2*eps -eps \n", "\n", "\n", " initial_nn_params=nm.append(nm.append(initial_Theta1.flatten(), initial_Theta2.flatten()), initial_Theta3.flatten() )\n", " \n", " ## let's compute the cost function and its gradient with the initial_Theta's\n", " \n", " #nnCostFunction(initial_nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lam)\n", " nnCostFunction2(initial_nn_params, input_layer_size, hidden_layer_size1,hidden_layer_size2, num_labels, X, y, lam)\n", "\n", " \n", " #res2=opti.minimize(nnCostFunction, initial_nn_params, jac=True, args=(input_layer_size, hidden_layer_size, num_labels, X, y, lam), method=\"L-BFGS-B\", tol=1e-6, options={'disp':True, 'maxiter':300, 'gtol':1e-6})\n", " res2=opti.minimize(nnCostFunction2, initial_nn_params, jac=True, args=(input_layer_size, hidden_layer_size1,hidden_layer_size2, num_labels, X, y, lam), method=\"L-BFGS-B\", tol=1e-6, options={'disp':False, 'maxiter':500, 'gtol':1e-6})\n", " \n", " N_elem_theta1=hidden_layer_size1*(1+input_layer_size)\n", " N_elem_theta2=(hidden_layer_size1+1) * hidden_layer_size2\n", " N_elem_theta3=(hidden_layer_size2+1) * num_labels\n", "\n", "\n", " Theta1=(res2.x)[0:N_elem_theta1].reshape((hidden_layer_size1, input_layer_size+1))\n", " Theta2=(res2.x)[N_elem_theta1:N_elem_theta1+N_elem_theta2].reshape((hidden_layer_size2, hidden_layer_size1+1))\n", " Theta3=(res2.x)[N_elem_theta1+N_elem_theta2:].reshape(( num_labels, hidden_layer_size2+1))\n", "\n", " return Theta1, Theta2, Theta3" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f0521e6a-15cc-faa8-66ca-7951a8153ef6" }, "outputs": [], "source": [ "## Main script\n", "# Loading the dataset\n", "A=pd.read_csv('../input/train.csv').values\n", "B=pd.read_csv('../input/test.csv').values\n", "\n", "## defining the target and the feature matrix.\n", "X=(A[:, 1:]-128)/128.\n", "B=(B[:,:]-128)/128.\n", "y=nm.int32(A[:,0])\n", "\n", "N=42000\n", "X_train=X[:N,:]\n", "y_train=y[:N]\n", "\n", "X_val=X[N:,:]\n", "y_val=y[N:]\n", "\n", "## Plot one digit for illustration\n", "id=244\n", "plt.figure(1)\n", "plt.clf()\n", "plt.imshow(X[id, :].reshape(28, 28), cmap=\"Greys\")\n", "plt.title('True value = {}'.format(y[id]))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a2d96b2c-564d-0d56-1635-03de07b63083" }, "outputs": [], "source": [ "lam=0.2\n", "\n", "## The next line launches the training, but this home-made implementation takes too much time. \n", "## Hence the line is commented.\n", "\n", "##T1, T2,T3= train2(X_train, y_train, 15,7, lam)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }