{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"### Input Processing\nimport pandas as pd\nimport numpy as np\n\n\nnp.random.seed(42)\ndata = pd.read_csv('../input/Kannada-MNIST/train.csv')\n\ndata['row_number'] = range(0,data.shape[0])\n\n\nprint(data.shape)\ndata = data.sample(frac=1,random_state=42)\ntmp = pd.DataFrame()\nfor label in range(10):\n    if label==0:\n        tmp = data[data['label']==label].head(600)\n    else:\n        temp = data[data['label']==label].head(600)\n        tmp = pd.concat([tmp,temp])\ndata_train = tmp\nrow_numbers_in_train_set = tmp['row_number'].values\ntest_set = data.loc[~data['row_number'].isin(row_numbers_in_train_set)]\nprint(data_train[['label']].groupby('label').size().reset_index())\n\none_hot = pd.get_dummies(data_train['label'].unique())\none_hot['label'] = one_hot.index\n\ndata_train = pd.merge(data_train,one_hot)\n#data = data.drop('label',axis=1)\ndata_test = test_set.sample(frac=1)\n\ntmp = pd.DataFrame()\nfor label in range(10):\n    if label==0:\n        tmp = data_test[data_test['label']==label].head(120)\n    else:\n        temp = data_test[data_test['label']==label].head(120)\n        tmp = pd.concat([tmp,temp])\ndata_test = tmp\n#data_test = data_test.head(500)\ndata_test = pd.merge(data_test,one_hot)\ndata_train.drop('label',axis=1,inplace=True)\n\ndata_test.drop('label',axis=1,inplace=True)\n\n## Create the train and test set\nX_train = np.array(data_train.drop([0,1,2,3,4,5,6,7,8,9,'row_number'],axis=1).values)/255\n#X_train = (X_train - np.mean(X_train)) / np.std(X_train)\ny_train = np.array(data_train[[0,1,2,3,4,5,6,7,8,9]].values)\nX_test = np.array(data_test.drop([0,1,2,3,4,5,6,7,8,9,'row_number'],axis=1).values)/255\n#X_test = (X_test - np.mean(X_test)) / np.std(X_test)\ny_test = np.array(data_test[[0,1,2,3,4,5,6,7,8,9]].values)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def maxpool(input_matrix,stride=2,filter_h=2,filter_w = 2,padding=0):\n    relu = np.ones((input_matrix.shape[1],filter_h,filter_w))\n    result_h = int((input_matrix.shape[2]-filter_h+padding)/stride)+1\n    result_w = int((input_matrix.shape[3]-filter_w+padding)/stride)+1\n    relu_result = np.zeros((input_matrix.shape[0],input_matrix.shape[1],result_h,result_w))\n    max_indices_for_result_relu = []\n    for image in range(input_matrix.shape[0]):\n        for filter_position in range(relu.shape[0]):\n            image_selected = input_matrix[image,:,:,:]\n            #result = []\n            filter_selected = relu[filter_position,:,:]\n            image_selected = image_selected[filter_position,:,:]\n            row = -1\n            column = -1\n            for i in range(0,image_selected.shape[0],stride):\n                image_rectangle = image_selected[i:i+filter_h,:]\n                if image_rectangle.shape[0]<filter_h:\n                    continue\n                else:\n                    row = row+1\n                    if row>result_h-1:\n                        row = 0\n                    for j in range(0,image_rectangle.shape[1],stride):\n                        image_portion = image_rectangle[:,j:j+filter_w]\n                        if image_portion.shape[1]<filter_w:\n                            continue\n                        else:\n                            column = column+1\n                            if column>result_w-1:\n                                column = 0\n                            convolution_result = np.multiply(filter_selected,image_portion)\n                            convolution_result[convolution_result<0] = 0\n                            i_max,j_max = np.unravel_index(convolution_result.argmax(),convolution_result.shape)\n                            i_max=i_max+i\n                            j_max = j_max+j\n                            \n                            max_indices_for_result_relu.append((image,filter_position,i_max,j_max,image,filter_position,row,column))                                          \n                            relu_result[image,filter_position,row,column] = np.max(convolution_result)\n                                \n                                \n            #result = np.array(result)\n            #result = np.ravel(result)\n            \n            #relu_result[image,filter_position,:,:] = result.reshape(result_h,result_w)\n    return(relu_result,max_indices_for_result_relu)\n\n\n        \n\ndef maxpool_reshape(shape_,input_matrix):\n    maxpool_reshape = np.zeros(shape_)\n    for row in range(input_matrix.shape[0]):\n        temp = input_matrix[row,:,:,:]\n        temp = np.ravel(temp)\n        maxpool_reshape[:,row] = temp\n    return(maxpool_reshape)\n    \n    \n    ### Function to reshape maxpool output to dense layer\ndef delta_conv_filter(filter_,input_matrix,delta_matrix,stride=1):\n    conv_delta = np.zeros(filter_.shape)\n    filter_h = filter_.shape[2]\n    filter_w = filter_.shape[3]\n    delta_x_conv = delta_matrix\n    X = input_matrix\n    for image in range(delta_x_conv.shape[0]):\n        delta_image = delta_x_conv[image,:,:,:]\n        input_image = X[image,:,:,:]\n        for channel in range(delta_image.shape[0]):\n            delta_image_channel = delta_image[channel,:,:]\n            delta_values = np.ravel(delta_image_channel)\n            for values in delta_values:\n                #print(values)\n                #### Get chunks from input matrix with same shape as filter\n                for row in range(0,input_image.shape[1],stride):\n                    image_rectangle = input_image[:,row:row+filter_h,:]\n                    if image_rectangle.shape[1]<filter_h:\n                        continue\n                    else:\n                        for column in range(0,image_rectangle.shape[2],stride):\n                            image_square = image_rectangle[:,:,column:column+filter_w]\n                            #print(image_square.shape)\n                            if image_square.shape[2]<filter_w:\n                                continue\n                            else:\n                                conv_delta[channel,:,:,:] +=(values*image_square)   \n    return(conv_delta)\n                       \ndef im2col(X,conv1, stride, pad):\n    \"\"\"\n        Transforms our input image into a matrix.\n\n        Parameters:\n        - X: input image.\n        - HF: filter height.\n        - WF: filter width.\n        - stride: stride value.\n        - pad: padding value.\n\n        Returns:\n        -cols: output matrix.\n    \"\"\"\n    # Padding\n    X_padded = np.pad(X, ((0,0), (0,0), (pad, pad), (pad, pad)), mode='constant')\n    X = X_padded\n    new_height = int((X.shape[2]+(2*pad)-(conv1.shape[2]))/stride)+1\n    new_width =  int((X.shape[3]+(2*pad)-(conv1.shape[3]))/stride)+1\n    im2col_vector = np.zeros((X.shape[1]*conv1.shape[2]*conv1.shape[3],new_width*new_height*X.shape[0]))\n    c = 0\n    for position in range(X.shape[0]):\n\n        image_position = X[position,:,:,:]\n        for height in range(0,image_position.shape[1],stride):\n            image_rectangle = image_position[:,height:height+conv1.shape[2],:]\n            if image_rectangle.shape[1]<conv1.shape[2]:\n                continue\n            else:\n                for width in range(0,image_rectangle.shape[2],stride):\n                    image_square = image_rectangle[:,:,width:width+conv1.shape[3]]\n                    if image_square.shape[2]<conv1.shape[3]:\n                        continue\n                    else:\n                        im2col_vector[:,c:c+1]=image_square.reshape(-1,1)\n                        c = c+1         \n            \n    return(im2col_vector)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.T\ny_train = y_train.T\n\nX_test = X_test.T\ny_test = y_test.T\n\ndef initializeFilter(size, scale = 1.0):\n    stddev = scale/np.sqrt(np.prod(size))\n    return np.random.normal(loc = 0, scale = stddev, size = size)\n\ndef sigmoid(x):\n    return(1./(1+np.exp(-x)))\n\ndef softmax(x): \n    \"\"\"Compute softmax values for each sets of scores in x.\"\"\" \n\n    e_x = np.exp(x - np.max(x)) \n\n    return (e_x / e_x.sum(axis=0)) \ndef ReLU(x):\n    return x * (x > 0)\n\ndef dReLU(x):\n    return 1. * (x > 0)\n\nimport random\nnp.random.seed(42)\n#conv1 = np.array([[[10,10,10],[0,0,0],[-10,-10,-10]],[[-10,0,10],[-10,0,10],[-10,0,10]]]).reshape(2,1,3,3)\nconv1 = np.random.randn(3,1,5,5) * np.sqrt(1. / 5)\nw1 = np.random.rand(60,12*12*3)/np.sqrt(12*12*3)\nb0 = np.zeros((60,1))/np.sqrt(12*12*3)\nw2 = np.random.rand(10,60)/np.sqrt(60)\nb1 = np.zeros((10,1))/np.sqrt(30)\nloss=[]\nbatches = 100\n\nlr = 0.01\nbatch_size = 200\nbeta = 0.9\ncount = 0\nepochs = 5\n    \nX_train_reshape = np.zeros((X_train.shape[1],1,28,28))\n\nfor i in range(X_train.shape[1]):\n    temp = X_train[:,i]\n    temp = np.ravel(temp)\n    temp = temp.reshape(28,28)\n    X_train_reshape[i,0,:,:] = temp\n    \nX_train= X_train_reshape  \n\nX_test_reshape = np.zeros((X_test.shape[1],1,28,28))\n\nfrom matplotlib import pyplot as plt\n\nfor i in range(X_test.shape[1]):\n    temp = X_test[:,i]\n    temp = np.ravel(temp)\n    temp = temp.reshape(28,28)\n#     plt.plot(temp)\n#     plt.show()\n#     plt.pause(0.5)\n    X_test_reshape[i,0,:,:] = temp\n\nX_test= X_test_reshape  \n\nprint(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)\n    \n    \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 100\nbatches = int(X_train.shape[0]/batch_size)\nprint(batches)\nepochs = 100\nlr = 0.001\nloss_weight_dict = {\n    \n}\nimport time\n\nX_conv = 0\n\n### Implementing Adam Optimizer\n\nbeta1 = 0.9\nbeta2 = 0.995\nmomentum_w1 = 0\nmomentum_w2 = 0\nmomentum_b0 = 0\nmomentum_b1 = 0\nmomentum_conv1 = 0\nvelocity_w1 = 0\nvelocity_w2 = 0\nvelocity_b0 = 0\nvelocity_b1 = 0\nvelocity_conv1 = 0\n\n\n\n### Forward Pass\nfor i in range(epochs):\n    permutation = np.random.permutation(X_train.shape[0])\n    X_train_shuffled = X_train[permutation,:,:,:]\n    Y_train_shuffled = y_train[:, permutation]\n#     beta1 = 0.9\n#     beta2 = 0.995\n#     momentum_w1 = 0\n#     momentum_w2 = 0\n#     momentum_b0 = 0\n#     momentum_b1 = 0\n#     momentum_conv1 = 0\n#     velocity_w1 = 0\n#     velocity_w2 = 0\n#     velocity_b0 = 0\n#     velocity_b1 = 0\n#     velocity_conv1 = 0\n\n    start = time.process_time()\n    for j in range(batches):\n        #print(conv1)\n        begin = j * batch_size\n        \n        end = min(begin + batch_size, X_train.shape[0] - 1)\n        if begin>=end:\n            continue\n        X = X_train_shuffled[begin:end,:,:,:]\n        Y = Y_train_shuffled[:, begin:end]\n        ## Get batch size\n        m_batch = end - begin\n        ## Pass input through convolutional Layer\n        X_im2col = im2col(X=X,conv1=conv1,pad=0,stride=1)\n        conv1_col = conv1.reshape(conv1.shape[0],-1)\n        X_result = conv1_col@X_im2col\n        X_result = np.array(np.hsplit(X_result, m_batch)).reshape((m_batch,conv1.shape[0],24,24))\n        X_conv = ReLU(X_result)\n        ### Pass output of convolutional layer to maxpooling layer\n        X_maxpool,max_index = maxpool(input_matrix=X_conv)\n        ### Get shape to flatten the maxpool output\n        flattened_shape = X_maxpool.shape[1]*X_maxpool.shape[2]*X_maxpool.shape[3]\n        ### Re-shape the maxpool output\n        X_maxpool_reshape = maxpool_reshape(shape_=(flattened_shape,m_batch),input_matrix=X_maxpool)\n        ## Pass the re-shaped (flattened) layer to sigmoid function\n        x1 = ReLU(w1@X_maxpool_reshape+b0)\n        ## Pass the output of the sigmoid layer to softmax layer for classification output\n        x2 = softmax(w2@x1+b1)\n        #print('Time Taken Forward Prop : ',time.process_time()-start)\n        ### Calculate Deltas\n        delta_2 = (x2-Y)\n        #np.multiply(x1,1-x1)\n        delta_1 = np.multiply(w2.T@delta_2,dReLU(x1))\n        delta_maxpool_reshape = np.multiply(w1.T@delta_1,1)\n        delta_maxpool = np.zeros(X_maxpool.shape)\n        #print(delta_relu.shape)\n        #### Re-shape maxpool delta create function\n        #for image in range(delta_maxpool_reshape.shape[1]):\n        delta_maxpool[0:m_batch,:,:,:] = delta_maxpool_reshape[:,0:m_batch].ravel().reshape(m_batch,X_maxpool.shape[1],X_maxpool.shape[2],X_maxpool.shape[3])\n        #print('Time Taken For Re-shape Maxpool Delta : ',time.process_time()-start)    \n        ## Propagate Delta to convolutional Layer from Max Layer\n        delta_x_conv = np.zeros((X_conv.shape))\n        for p in max_index:\n            \n            delta_x_conv[p[0],p[1],p[2],p[3]] = delta_maxpool[p[4],p[5],p[6],p[7]]\n        \n        #print('Time Taken to propagate delta for re-shape max pool delta : ',time.process_time()-start)\n        \n        dout = np.multiply(delta_x_conv,dReLU(X_conv))\n        dout_test= np.zeros((dout.shape[1],dout.shape[0]*dout.shape[2]*dout.shape[3]))\n        dout_test = np.zeros((delta_x_conv.shape[1],delta_x_conv.shape[0]*delta_x_conv.shape[2]*delta_x_conv.shape[3]))\n        for channel in range(delta_x_conv.shape[1]):\n            delta_x_conv_channel = delta_x_conv[:,channel,:,:]\n            dout_test[channel:channel+1,:] = np.ravel(delta_x_conv_channel)\n        dw_col = dout_test @ X_im2col.T\n        #dconv1 = dw_col.reshape((dw_col.shape[0],1,3, 3))\n        \n        \n        \n        \n        #print('Done!')\n        if i==0 :\n            dW1 = delta_1@X_maxpool_reshape.T\n            dW2 = delta_2@x1.T\n            db0 = np.sum(delta_1,axis=1,keepdims=True)\n            db1 = np.sum(delta_2,axis=1,keepdims=True)\n            dconv1 = dw_col.reshape((dw_col.shape[0],1,5, 5))\n            momentum_w1 = beta1*momentum_w1 + ((1-beta1)*dW1)\n            momentum_w2 = beta1*momentum_w2 + ((1-beta1)*dW2)\n            momentum_b0 = beta1*momentum_b0 + ((1-beta1)*db0)\n            momentum_b1 = beta1*momentum_b1 + ((1-beta1)*db1)\n            momentum_conv1 = beta1*momentum_conv1 + ((1-beta1)*dconv1)\n            velocity_w1 = beta2*velocity_w1 + ((1-beta2)*dW1**2)\n            velocity_w2 = beta2*velocity_w2 + ((1-beta2)*dW2**2)\n            velocity_b0 = beta2*velocity_b0 + ((1-beta2)*db0**2)\n            velocity_b1 = beta2*velocity_b1 + ((1-beta2)*db1**2)\n            velocity_conv1 = beta2*velocity_conv1 + ((1-beta2)*dconv1**2)\n            \n            \n            #print('Time Taken to calculate dconv1 : ',time.process_time()-start)\n            ### Create dconv1\n        else:\n            \n            ## Create Conv1\n            dW1 = delta_1@X_maxpool_reshape.T\n            dW2 = delta_2@x1.T\n            db0 = np.sum(delta_1,axis=1,keepdims=True)\n            db1 = np.sum(delta_2,axis=1,keepdims=True)\n            dconv1 = dw_col.reshape((dw_col.shape[0],1,5, 5))\n            #print('Time Taken to calculate dconv1 : ',time.process_time()-start)\n            momentum_w1 = beta1*momentum_w1 + ((1-beta1)*dW1)\n            momentum_w2 = beta1*momentum_w2 + ((1-beta1)*dW2)\n            momentum_b0 = beta1*momentum_b0 + ((1-beta1)*db0)\n            momentum_b1 = beta1*momentum_b1 + ((1-beta1)*db1)\n            momentum_conv1 = beta1*momentum_conv1 + ((1-beta1)*dconv1)\n            velocity_w1 = beta2*velocity_w1 + ((1-beta2)*(dW1**2))\n            velocity_w2 = beta2*velocity_w2 + ((1-beta2)*(dW2**2))\n            velocity_b0 = beta2*velocity_b0 + ((1-beta2)*(db0**2))\n            velocity_b1 = beta2*velocity_b1 + ((1-beta2)*(db1**2))\n            velocity_conv1 = beta2*velocity_conv1 + ((1-beta2)*(dconv1**2))\n            #print(time.process_time()-start)\n            \n\n\n        w1 = w1 - ((lr*momentum_w1)/(np.sqrt(velocity_w1)+0.0000001))\n        b0 = b0 - ((lr*momentum_b0)/(np.sqrt(velocity_b0)+0.0000001))\n        w2 = w2 - ((lr*momentum_w2)/(np.sqrt(velocity_w2)+0.0000001))\n        b1 = b1 - ((lr*momentum_b1)/(np.sqrt(velocity_b1)+0.0000001))\n        conv1 = conv1 -((lr*momentum_conv1)/(np.sqrt(velocity_conv1)+0.0000001))\n    \n    #lr = lr*np.exp(-0.1*i)\n    \n    X_im2col = im2col(X=X_train,conv1=conv1,pad=0,stride=1)\n    conv1_col = conv1.reshape(conv1.shape[0],-1)\n    X_result = conv1_col@X_im2col\n    X_result = np.array(np.hsplit(X_result, X_train.shape[0])).reshape((X_train.shape[0],conv1.shape[0],24,24))\n    X_train_ = ReLU(X_result)    \n    #X_train_ = convolution(input_matrix=X_train,filter_=conv1)\n    X_train_,max_index = maxpool(input_matrix = X_train_)\n    flattened_shape = X_train_.shape[1]*X_train_.shape[2]*X_train_.shape[3]\n    X_train_reshape = np.zeros((flattened_shape,X_train_.shape[0]))\n    for row in range(X_train_.shape[0]):\n        temp = X_train_[row,:,:,:]\n        temp = np.ravel(temp)\n        X_train_reshape[:,row] = temp\n    \n    X_train_ = X_train_reshape\n    x1 = ReLU(w1@X_train_+b0)\n    x2_train = softmax(w2@x1+b1)\n    x2_train_df = pd.DataFrame(x2_train)\n    x2_train_df = (x2_train_df == x2_train_df.max()).astype(int)\n    x2_train_df = x2_train_df.transpose()\n    x2_train_df = pd.merge(x2_train_df,one_hot)\n    x2_train_df = x2_train_df[['label']]\n    y_train_df = pd.merge(pd.DataFrame(y_train.T),one_hot)\n    x2_train_df['label_actual'] = y_train_df['label']\n    train_accuracy = np.sum(x2_train_df['label_actual']==x2_train_df['label'])/x2_train_df.shape[0]\n\n    \n    print('Training Loss...')\n    print(-np.mean(np.multiply(y_train,np.log(x2_train))))\n    \n#     if -np.mean(np.multiply(y_train,np.log(x2_train))) < 0.01:\n#         lr = 0.001\n#     if -np.mean(np.multiply(y_train,np.log(x2_train))) < 0.005:\n#         lr = 0.001\n    \n    add_loss = {\n        'loss' : -np.mean(np.multiply(y_train,np.log(x2_train))),\n        'weight_1' : w1,\n        'weight_2':w2,\n        'conv1':conv1,\n        'b0' : b0,\n        'b1': b1,\n        'train_accuracy': train_accuracy\n    }\n    \n    \n    X_im2col = im2col(X=X_test,conv1=conv1,pad=0,stride=1)\n    conv1_col = conv1.reshape(conv1.shape[0],-1)\n    X_result = conv1_col@X_im2col\n    X_result = np.array(np.hsplit(X_result, X_test.shape[0])).reshape((X_test.shape[0],conv1.shape[0],24,24))\n    X_test_ = ReLU(X_result)\n    \n    #X_test_ = convolution(input_matrix=X_test,filter_=conv1)\n    X_test_,max_index = maxpool(input_matrix=X_test_)\n    flattened_shape = X_test_.shape[1]*X_test_.shape[2]*X_test_.shape[3]\n    X_test_reshape = np.zeros((flattened_shape,X_test_.shape[0]))\n    for row in range(X_test_.shape[0]):\n        temp = X_test_[row,:,:,:]\n        temp = np.ravel(temp)\n        X_test_reshape[:,row] = temp\n    \n    X_test_ = X_test_reshape\n    x1 = ReLU(w1@X_test_+b0)\n    x2_test = softmax(w2@x1+b1)\n    x2_test_df = pd.DataFrame(x2_test)\n    x2_test_df = (x2_test_df == x2_test_df.max()).astype(int)\n    x2_test_df = x2_test_df.transpose()\n    x2_test_df = pd.merge(x2_test_df,one_hot)\n    x2_test_df = x2_test_df[['label']]\n    y_test_df = pd.merge(pd.DataFrame(y_test.T),one_hot)\n    x2_test_df['label_actual'] = y_test_df['label']\n    test_accuracy = np.sum(x2_test_df['label_actual']==x2_test_df['label'])/x2_test_df.shape[0]\n    print('Epoch: ',i)\n    print('Time Taken for Epoch', i)\n    print(time.process_time() - start)\n    print('Testing Accuracy :',test_accuracy)\n    print('Training Accuracy :',train_accuracy)\n    print('----------------------------------------')\n    \n    \n    \n#     print('Testing Loss...')\n#     print(-np.mean(np.multiply(y_test,np.log(x2))))\n    \n    add_loss['testing_loss'] = -np.mean(np.multiply(y_test,np.log(x2_test)))\n    add_loss['test_accuracy'] = test_accuracy\n    loss_weight_dict[count] = add_loss\n    count = count + 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss = []\n\nfor i in range(len(loss_weight_dict)):\n    loss.append(loss_weight_dict[i]['loss'])\nimport matplotlib.pyplot as plt\nplt.plot(loss)\nplt.xlabel('Epochs')\nplt.ylabel('loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_accuracy = []\n\nfor i in range(len(loss_weight_dict)):\n    train_accuracy.append(loss_weight_dict[i]['train_accuracy'])\nimport matplotlib.pyplot as plt\nplt.plot(train_accuracy)\nplt.xlabel('Epochs')\nplt.ylabel('Training Accuracy')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_accuracy = []\n\nfor i in range(len(loss_weight_dict)):\n    test_accuracy.append(loss_weight_dict[i]['test_accuracy'])\nimport matplotlib.pyplot as plt\nplt.plot(test_accuracy)\nplt.xlabel('Epochs')\nplt.ylabel('Testing Accuracy')\nplt.show()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}