{"cells":[{"metadata":{"id":"iQiMX0R3kLW4","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#load modules\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport math\n","execution_count":null,"outputs":[]},{"metadata":{"id":"b_0K_DGAkR8R","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#get data\ntrain = pd.read_csv(\"../input/train.csv\" )\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"id":"_TZ_ljK1kb09","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#split training set into input (x_train) and output (y_train)  \nx_train = train.iloc[: , 1:  ].values.T\ny_train = train.iloc [ : , 0 ].values\nx_test = test.iloc[: , : ].values.T\ny_train = y_train.reshape(-1 , 1)\nprint(y_train.shape )","execution_count":null,"outputs":[]},{"metadata":{"id":"iR98fyPNkeT0","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#normalizing \nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\nx_train = x_train/255\nx_test = x_test/255\n","execution_count":null,"outputs":[]},{"metadata":{"id":"3x_Q5Jh5kgbw","colab_type":"code","outputId":"352a3527-22d0-4524-b456-212eda7b2999","colab":{"base_uri":"https://localhost:8080/","height":224},"trusted":true},"cell_type":"code","source":"#one_hot_encoding\nfrom sklearn.preprocessing import OneHotEncoder\none_hot = OneHotEncoder(sparse=False , categories = 'auto' )\ny_train = one_hot.fit_transform(y_train)\ny_train = y_train.T\nprint(y_train.shape )\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"V7DdosdCklCM","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#weight intialization\ndef init_params (l_dims) :  \n  params = {}\n  L= len(l_dims) \n  for l in range(1 , L ) :\n    params['w'+ str(l)] = np.random.randn(l_dims[l] , l_dims[l-1]) * np.sqrt(1. / l_dims[l-1] )\n    params['b'+ str(l)] = np.zeros ((l_dims[l] , 1)) * np.sqrt(1. /l_dims[l-1])        \n  \n  return params \n  ","execution_count":null,"outputs":[]},{"metadata":{"id":"gZIL2eNDliFx","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def init_adam(l_dims) :\n  v = {}\n  s = {}\n  L= len(l_dims) \n  for l in range(1 , L ) :\n      v[\"dw\" + str(l)] = np.zeros(( l_dims[l] ,l_dims[l-1] ))\n      v[\"db\" + str(l)] = np.zeros((l_dims[l] , 1) )     \n      s[\"dw\" + str(l)] = np.zeros(( l_dims[l] ,l_dims[l-1] ))\n      s[\"db\" + str(l)] = np.zeros((l_dims[l] , 1) )      \n  \n  return v ,s","execution_count":null,"outputs":[]},{"metadata":{"id":"EjNU9-fE_oTP","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def relu (x) :  \n     return x * (x > 0)","execution_count":null,"outputs":[]},{"metadata":{"id":"nS3C9Cyc_sbY","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#derivative of relu function\ndef reLuD(x):\n    return 1. * (x > 0)","execution_count":null,"outputs":[]},{"metadata":{"id":"_AvbZXubCOUQ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def softmax(x):\n    e_x = np.exp(x - np.max(x))\n    return e_x / e_x.sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"id":"5OmVVUM9lqiV","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def compute_loss(Y, Y_hat):\n    \n    L_sum = np.sum(np.multiply(Y, np.log(Y_hat)))\n    m = Y.shape[1]\n    L = -(1./m) * L_sum\n\n    return L","execution_count":null,"outputs":[]},{"metadata":{"id":"WPsEW_CjlxMs","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def forward_prop ( x , params , l_dims , keep_prop ) :\n  L = len(l_dims)\n  cache = {}\n  cache [\"a0\"] = x \n  for l in range (1 , L) :\n    z = np.dot( params['w' + str(l)] , cache['a' + str(l-1)] ) + params['b' + str(l)] \n    if( l == L-1 ) :\n      a = softmax(z)\n    else :  \n      a = relu(z)\n      #Dropout technique to prevent overfitting \n      d = np.random.rand( a.shape[0] ,  a.shape[1])\n      d = d < keep_prop \n      a = a*d \n      a = a/keep_prop\n    \n    cache['z' + str(l)] = z \n    cache['a' + str(l)] = a \n    cache['d' +str(l) ] = d \n    \n  return  a, cache ","execution_count":null,"outputs":[]},{"metadata":{"id":"8bFmvQgwmVJ6","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"\ndef back_prop ( a , y , cache , params , l_dims , keep_prop ) :\n    grads = {}\n    L = len(l_dims)\n    m = Y.shape[1]\n    dz = a - Y\n    for l in range(L-1 , 0 , -1 ) :\n      dw = 1/m * np.dot ( dz , cache['a' + str(l-1)].T ) \n      db = 1/m * np.sum ( dz , axis = 1 , keepdims = True ) \n      if( l > 1) :\n        da = np.dot(params['w' +str(l)].T , dz )\n        da = da*cache[\"d\"+str(l-1)] \n        da = da/keep_prop        \n        dz =  da * reLuD(cache['z' + str(l-1)]) \n      grads[\"dw\" + str(l)] = dw \n      grads[\"db\" + str(l)] = db \n\n    return grads","execution_count":null,"outputs":[]},{"metadata":{"id":"d2TD6vHW5jZL","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#update weights with Adam algorithm \ndef update_params( l_dims , params , grads , v , s , lr , beta1 , beta2 ,t ) :\n  v_c = {} \n  s_c = {}\n  for l in range ( 1 , len(l_dims) ) :\n    v['dw' + str(l) ] = beta1*v['dw' + str(l)] + (1-beta1)*grads['dw'+ str(l) ]\n    v['db' + str(l) ] = beta1*v['db' + str(l)] + (1-beta1)*grads['db'+ str(l) ]\n    v_c[\"dw\" + str(l)] = v[\"dw\" + str(l)] / (1 - np.power(beta1, t))\n    v_c[\"db\" + str(l)] = v[\"db\" + str(l)] / (1 - np.power(beta1, t)) \n    \n    s[\"dw\" + str(l)] = beta2*s[\"dw\" + str(l)] + (1-beta2) * np.power(grads['dw'+str(l)] , 2 )\n    s[\"db\" + str(l)] = beta2*s[\"db\" + str(l)] + (1-beta2) * np.power (grads['db' + str(l)] , 2 ) \n    s_c[\"dw\" + str(l)] = s[\"dw\" + str(l)] / (1 - np.power(beta2, t))\n    s_c[\"db\" + str(l)] = s[\"db\" + str(l)] / (1 - np.power(beta2, t)) \n    params['w' + str(l)] = params['w' + str(l)] - lr * v_c[\"dw\" + str(l)] / np.sqrt(s_c[\"dw\" + str(l)] + 1e-8)\n    params['b' + str(l)] = params['b' + str(l)] - lr * v_c[\"db\" + str(l)] / np.sqrt(s_c[\"db\" + str(l)] + 1e-8)\n \n   \n\n  return params ,v, s    ","execution_count":null,"outputs":[]},{"metadata":{"id":"pi_E9qLMuNak","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def accurcy (y_p , y ) :   \n    m = y_p.shape[1]\n    sum = 0\n    for i in range(0,m) :\n      if( np.argmax(y_p[: , i])  == np.argmax( y[: , i] ) ) :\n         sum = sum+1\n    return sum/m","execution_count":null,"outputs":[]},{"metadata":{"id":"ZUhYTvSrm5Q2","colab_type":"code","outputId":"2cc8a4c8-2c4f-41f3-fc3c-9d824ca876be","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"l_dims = [x_train.shape[0] ,256, 10] #NN architecture 2\nm = x_train.shape[1] #num of training examples \nparams = init_params(l_dims) \nv,s = init_adam(l_dims)\nkeep_prop = 0.4 # Dropout parameter \nbatch_size = 32 \nepochs = 50 #num of epochs  \nlr = 0.001 # learning rate \n\nfor i in range(epochs):\n\n    # shuffle training set\n    permutation = np.random.permutation(m)\n    X_train_shuffled = x_train[:, permutation]\n    Y_train_shuffled = y_train[:, permutation]\n     \n    for j in range(math.ceil(m/batch_size )) :\n\n        # get mini-batch\n        begin = j * batch_size\n        end = min(begin + batch_size, m - 1)\n        X = X_train_shuffled[:, begin:end]\n        Y = Y_train_shuffled[:, begin:end]\n        m_batch = end - begin\n\n        # forward and backward\n        a,cache = forward_prop(X, params , l_dims , keep_prop )\n        grads = back_prop(a, Y, cache ,params, l_dims , keep_prop)\n\n        # updat parameters with gradient descent\n        params ,v,s = update_params(l_dims, params , grads , v , s , lr , 0.9, 0.999 ,j+1 )\n     \n    # compute cost on training set \n    a , cache = forward_prop(x_train, params,l_dims , 1)\n    train_loss = compute_loss(y_train, a)\n\n    print( \"epoch \" + str(i) + \" loss = \" , train_loss )","execution_count":null,"outputs":[]},{"metadata":{"id":"Efp-6cTl5LTz","colab_type":"code","outputId":"df4f17a8-4d9c-42b4-ef8a-ca50d6c3b476","colab":{"base_uri":"https://localhost:8080/","height":51},"trusted":true},"cell_type":"code","source":"#compute accuracy on training set \na ,cache = forward_prop(x_train, params,l_dims ,1)\nprint(accurcy(a , y_train))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#predict\ny_hat ,cache = forward_prop(x_test, params,l_dims,1 )\ny_pred = np.argmax(y_hat,axis=0) #label test data \nm_test = x_test.shape[1] #num of test set examples \noutput_file = \"submission.csv\"\nid = []\nfor i in range(m_test ) :\n    id.append(i+1)\n    \n    \nsubmission = pd.DataFrame({'ImageId': id , 'Label': y_pred})\nsubmission.to_csv('submission.csv', index=False)\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Untitled0.ipynb","version":"0.3.2","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"nbformat":4,"nbformat_minor":1}