{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom scipy import optimize\n# Input data files are available in the read-only \"../input/\" directory\nimport matplotlib.pyplot as plt\n\nimport os\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1/(1+np.exp(-x))\ndef sigder(x):\n    return sigmoid(x) * (1-sigmoid(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"            # we will be using a single layered neuran network for this implementation\n\ndef nnCost(nn_params,\n           input_layer_size,\n           hidden_layer_size,\n           num_labels,\n           X,y,lambda_=0):\n#     with the formula we have fabricated we can get the two weights of the neural network which\n#     we can write as theta1 and theta 2 which are concatenated in the nnparams\n#     thetai = nnparams[len(thetai-1): hiddenlayeri * (hiddenlayer(i-1) + 1) + len(theta(i-1))]\n    Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],\n                        (hidden_layer_size, (input_layer_size + 1)))\n\n    Theta2 = np.reshape(nn_params[(hidden_layer_size * (input_layer_size + 1)):],\n                        (num_labels, (hidden_layer_size + 1)))\n\n    # Setup some useful variables\n    m = y.size\n         \n    # we need to get the following parameters as the result of this function\n    J = 0\n    Theta1_grad = np.zeros(Theta1.shape)\n    Theta2_grad = np.zeros(Theta2.shape)\n\n\n    \n    a1 = np.concatenate([np.ones((m, 1)), X], axis=1)\n    \n    a2 = sigmoid(a1.dot(Theta1.T))\n    a2 = np.concatenate([np.ones((a2.shape[0], 1)), a2], axis=1)\n    \n    a3 = sigmoid(a2.dot(Theta2.T))\n    \n    y_matrix = y.reshape(-1)\n    y_matrix = np.eye(num_labels)[y_matrix]\n    \n    temp1 = Theta1\n    temp2 = Theta2\n    \n    # Add regularization term\n    \n    reg_term = (lambda_ / (2 * m)) * (np.sum(np.square(temp1[:, 1:])) + np.sum(np.square(temp2[:, 1:])))\n    \n    J = (-1 / m) * np.sum((np.log(a3) * y_matrix) + np.log(1 - a3) * (1 - y_matrix)) + reg_term\n    \n    # Backpropogation\n    \n    delta_3 = a3 - y_matrix\n    delta_2 = delta_3.dot(Theta2)[:, 1:] * sigder(a1.dot(Theta1.T))\n\n    Delta1 = delta_2.T.dot(a1)\n    Delta2 = delta_3.T.dot(a2)\n    \n    # Add regularization to gradient\n\n    Theta1_grad = (1 / m) * Delta1\n    Theta1_grad[:, 1:] = Theta1_grad[:, 1:] + (lambda_ / m) * Theta1[:, 1:]\n    \n    Theta2_grad = (1 / m) * Delta2\n    Theta2_grad[:, 1:] = Theta2_grad[:, 1:] + (lambda_ / m) * Theta2[:, 1:]\n \n    \n    grad = np.concatenate([Theta1_grad.ravel(), Theta2_grad.ravel()])\n    \n    return J, grad\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lets define our input now \ntrainfile = '../input/digit-recognizer/train.csv'\ntraindata = pd.read_csv(trainfile).to_numpy()\n\nhidden_layer_size = 49\nnum_labels = 10\ninput_layer_size = traindata[:,1:].shape[1]\nX,y = traindata[:,1:],traindata[:,0]\nm = y.size\n#lets show a random data from out traning set\nrand = np.random.randint(0,m)\nplt.imshow(255- X[rand].reshape(28,28) , cmap='gray')\ny[rand]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#now we get the randomized weights for the neural network\ndef randweightgenerator(L_in,L_out,epsilon = 0.12):\n    W = np.zeros((L_out, 1 + L_in))\n    W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon - epsilon\n    return W\ntheta1 = randweightgenerator(input_layer_size, hidden_layer_size)\ntheta2 = randweightgenerator(hidden_layer_size, num_labels)\n#now we set nn_params as a list of theta 1 and theta 2 \n\nnn_params = np.concatenate([theta1.ravel(), theta2.ravel()], axis=0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#now we can use the optimizer from the scipy library to minimize our weights \n#  the other way to optimize this would be to use the gradients and do gradient descent\noptions = {'maxiter':1000}\nlambda_ = 3\n\n#now we make a temp cost function \ncostfn = lambda p :nnCost(p,input_layer_size,hidden_layer_size,num_labels,X,y,lambda_)\n\nres = optimize.minimize(costfn,nn_params,jac=True ,method=\"TNC\",options=options)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"finalnnparams = res.x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"j,_ = nnCost(finalnnparams,input_layer_size,hidden_layer_size,num_labels,X,y,lambda_)\nprint('The final cost after traning is : ',j)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# now lets define a fucntion which will use the given weights and predict the digits\ndef predict(nn_params,X,input_layer_size):\n    Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],\n                        (hidden_layer_size, (input_layer_size + 1)))\n\n    Theta2 = np.reshape(nn_params[(hidden_layer_size * (input_layer_size + 1)):],\n                        (num_labels, (hidden_layer_size + 1)))\n    m = X.shape[0]\n    \n    a1 = np.concatenate([np.ones((m, 1)), X], axis=1)\n    \n    a2 = sigmoid(a1.dot(Theta1.T))\n    a2 = np.concatenate([np.ones((a2.shape[0], 1)), a2], axis=1)\n    \n    a3 = sigmoid(a2.dot(Theta2.T))\n#     now lets get the index of maximum valued probability from p\n    p = np.argmax(a3, axis = 1)\n    return p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load the testing data\ntestfile = '../input/digit-recognizer/test.csv'\ntestdata = pd.read_csv(testfile).to_numpy()\nX_test = testdata\ninput_layer_size = testdata.shape[1]\n\noutputs = predict(finalnnparams,X_test,input_layer_size)\nfinalres = outputs.tolist()\n\n#submission part\ndf = pd.DataFrame(finalres)\ndf= df.rename(columns={0:'Label'})\ndf['ImageId'] = [i+1 for i in range(outputs.size)]\ndf.set_index('ImageId',inplace=True)\ndf.to_csv('submit.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#now lets see some predictions\n# visualization of the predictions\nrand = np.random.randint(0,28000)\nplt.imshow(255- X_test[rand].reshape(28,28) , cmap='gray')\noutputs[rand]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}