{"cells":[{"metadata":{"_uuid":"53ee34cb-eab3-4dc8-abb6-19ca2a0df4a0","_cell_guid":"2415a881-a5bb-4553-b052-76b654de56f8","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d594c1e2-a0dc-4dc1-8546-f804f43a3f26","_cell_guid":"8d266262-078d-4d4e-bd40-a6b94ceaf288","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb719504-881a-4032-be1e-5f72a6db1b01","_cell_guid":"ba2d3435-8633-491f-8bfe-be171fb13437","trusted":true},"cell_type":"code","source":"train_labels = train_data.pop(\"label\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"869fe8ec-c34c-4cef-85dd-10c0dac8cb8c","_cell_guid":"dff4aba6-a7c9-4a74-9892-a7688b330821","trusted":true},"cell_type":"code","source":"# train_X=COMPLETE_X[:5000]\n# train_Y=(COMPLETE_Y[:5000])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03ea77d7-d76c-4d80-b773-2c25d5462ac2","_cell_guid":"e5dde093-fe89-4838-a23d-59336550f15a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7303ddde-b351-4fb6-ba19-6708e3e9099d","_cell_guid":"3dc79444-5be5-4906-acfd-ea69071abb75","trusted":true},"cell_type":"code","source":"def show_example(i):\n    image = train_data.values\n    temp=image[i].reshape((28,28))\n    plt.imshow(temp,cmap='gray')\n    plt.show()\n    \n    print(\"Label : \" + str(train_labels[i]))\n    \nshow_example(2)\nshow_example(23)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a1ee93b-5b8a-45c9-a144-ef5931823163","_cell_guid":"e64fed2d-8e3a-4a5d-86a4-5c575431aecd","trusted":true},"cell_type":"code","source":"def cleanXY(X,Y=None):\n    X=np.matrix(X.values,dtype=np.float128)\n    if Y is not None:\n        Y=pd.get_dummies(Y)\n        Y=np.matrix(Y.values,dtype=np.float128)\n    return X,Y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a59e65d8-0ac6-48a6-8745-c9ec025e76c8","_cell_guid":"c7915539-ccc7-489f-ada1-c10268f87619","trusted":true},"cell_type":"code","source":"def normalize(x):\n    mean_X=np.sum(x,axis=0).reshape(1,-1)/x.shape[0]\n    sd_X=np.sum(np.square(x),axis=0).reshape(1,-1)/x.shape[0]\n    norm_x=(x-mean_X.reshape(1,-1)  )/(np.sqrt(sd_X)+(10**-5)).reshape(1,-1)\n\n    return norm_x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d48ff83a-7540-4591-8415-fee611fd6115","_cell_guid":"075d112e-909f-4542-b444-9a4292e8afc7","trusted":true},"cell_type":"code","source":"print(train_data.shape)\nprint(train_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bee0498a-a9a7-4019-9283-1bb9d79be8da","_cell_guid":"6b117b7f-89be-4bfb-93ad-ba4a5626dc37","trusted":true},"cell_type":"code","source":"def init_params(*args):\n    np.random.seed(3)\n    params={}\n    for i in range(len(args)-1):\n        params[f\"W{str(i+1)}\"]= np.random.randn(args[i],args[i+1])*0.01#np.sqrt(1/n_0) # Xaviers init\n        params[f\"W{str(i+1)}\"]=np.array(params[f\"W{str(i+1)}\"],dtype=np.float128)\n    \n        params[f\"b{str(i+1)}\"]=np.zeros((args[i+1],1))\n        params[f\"b{str(i+1)}\"]=np.array(params[f\"b{str(i+1)}\"],dtype=np.float128)\n       \n    return params","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a372c5fc-0b26-4da2-9955-96ce0f9f74b3","_cell_guid":"334225c2-12c8-4dfa-893f-d9967d14430c","trusted":true},"cell_type":"code","source":"#test init_params\nparams=init_params(10,5)\ndisplay([k+\":\"+str(x.shape) for k,x in params.items()])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17c8ce2e-bac9-4a30-8aa7-3c363b61dd9d","_cell_guid":"f5f47bde-5dfc-4cbf-bee3-e012a1190261","trusted":true},"cell_type":"code","source":"def softmax(z):\n    z_exp = np.exp(z)\n    sum_z_exp = np.sum(z_exp,axis=1).reshape(-1,1)\n    lst=z_exp/sum_z_exp\n    return np.array(lst,dtype=np.float128)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"843f70df-4dba-46b1-ba32-2f757359e8b6","_cell_guid":"28e8d776-6846-4dc1-a6d7-84849ae454b6","trusted":true},"cell_type":"code","source":"'''Cost Function for 2 layer neural network'''\ndef cost(Y_pred,Y):\n    m=Y.shape[0]\n    logprobs = np.multiply(Y,np.log(Y_pred),dtype=np.float128)\n    cost = - (1/m)*((logprobs))     \n    return np.sum(cost)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f6547ac-0bb5-4db1-b4aa-4301ae2a59ec","_cell_guid":"82e4d007-5de1-4696-98b2-e3444b35335b","trusted":true},"cell_type":"code","source":"#Test cost\nA2=softmax(np.matrix([[0.02,0.23,0.94,0.001],[0.02,0.23,0.94,0.001],[0.02,0.23,0.94,0.001]]))\n\nY= np.matrix([[0, 1, 0, 0],[0, 1, 0, 0],[0, 1, 0, 0]])\nprint(A2.shape)\nprint(Y.shape)\nprint(cost(A2,Y))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5092c33c-01c0-4171-aedc-f7a1b027d301","_cell_guid":"51763c7d-8390-4d67-8a4f-a9ff0929a086","trusted":true},"cell_type":"code","source":"def forward_propogation(X,weights,num_of_layers):\n\n    m=X.shape[0]\n\n    cache_params={\"A0\":X}\n    for i in range(num_of_layers-1):\n        cache_params[\"Z\"+str(i+1)]=np.dot(cache_params[\"A\"+str(i)],weights[\"W\"+str(i+1)])+weights[\"b\"+str(i+1)].T        \n        cache_params[\"A\"+str(i+1)]=np.tanh(cache_params[\"Z\"+str(i+1)])\n\n        \n    cache_params[\"Z\"+str(num_of_layers)]=np.dot(cache_params[\"A\"+str(num_of_layers-1)],weights[\"W\"+str(num_of_layers)])+weights[\"b\"+str(num_of_layers)].T        \n    cache_params[\"A\"+str(num_of_layers)]=softmax(cache_params[\"Z\"+str(num_of_layers)])\n    \n    return cache_params[\"A\"+str(num_of_layers)],cache_params","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95bf654a-6486-40a5-a135-fd99c02d41cb","_cell_guid":"b6d6316f-cb8c-40a0-8159-04ca95b4f6e4","trusted":true},"cell_type":"code","source":"def back_propogation(Y_pre,Y,X,cache_params,params,num_of_layers,learning_rate=0.1):    \n    m = Y.shape[0]\n    updated_params={}\n    \n    updated_params[\"dZ\"+str(num_of_layers)]= Y_pre-Y\n    updated_params[\"dW\"+str(num_of_layers)]=(1/m)*(np.dot(updated_params[\"dZ\"+str(num_of_layers)].T,cache_params[\"A\"+str(num_of_layers-1)])).T\n    updated_params[\"db\"+str(num_of_layers)]=(1/m)*np.sum(updated_params[\"dZ\"+str(num_of_layers)].T,axis=1)\n    \n    for x in range(num_of_layers-1,0,-1):\n    \n        derivative=np.dot(params[\"W\"+str(x+1)],updated_params[\"dZ\"+str(x+1)].T).T\n        temp2=cache_params[\"Z\"+str(x)]\n        val1= 1/np.square(np.cosh(temp2))\n        updated_params[\"dZ\"+str(x)]=np.multiply(derivative,val1) \n    \n        updated_params[\"dW\"+str(x)]=(1/m)*(np.dot(updated_params[\"dZ\"+str(x)].T,cache_params[\"A\"+str(x-1)])).T\n        updated_params[\"db\"+str(x)]=(1/m)*np.sum(updated_params[\"dZ\"+str(x)].T,axis=1)\n    \n    for i in range(1,num_of_layers):\n        params[\"W\"+str(i)]=params[\"W\"+str(i)]-learning_rate* updated_params[\"dW\"+str(i)]\n        params[\"b\"+str(i)]=params[\"b\"+str(i)]-learning_rate* updated_params[\"db\"+str(i)]\n    return params","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ce10d6e-7e10-4e9b-ae89-920040ed0437","_cell_guid":"4e375b16-88a4-4d9d-86cc-9c143f1fe7e1","trusted":true},"cell_type":"code","source":"def show_accuracy(Y_expanded,predictions):\n    flags=np.argmax(Y_expanded,axis=1).reshape(1,-1)\n    preds=np.argmax(predictions,axis=1)\n# print(preds)\n\n\n    success=(flags==preds).sum()/len(preds)\n\n    print(\"Accuracy = \" +str(success*100)+\"%\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e6b20c7-275a-4100-8380-d1a56fcfc530","_cell_guid":"c6bfa666-f5c3-4822-8564-71d1558457e2","trusted":true},"cell_type":"code","source":"def train(X,Y,mini_batch_size=512,number_of_iter=100,hidden_layers=[]):\n    layer_dims=[X.shape[1]]+hidden_layers\n    layer_dims.append(Y.shape[1])\n    params=init_params(*layer_dims)\n    num_of_layers=len(layer_dims)-1\n    num_mini_batches = np.math.ceil(X.shape[0]/mini_batch_size)\n\n    cost_tracker = []\n    for mini_batch in range(num_mini_batches):\n        mini_X=normalize(X[mini_batch*mini_batch_size :(mini_batch+1)*mini_batch_size ])\n        mini_Y=Y[mini_batch*mini_batch_size :(mini_batch+1)*mini_batch_size]\n        mini_cost_tracker=[0]\n\n        decay_rate = 0.001 #((decay_rate)/2)\n        if mini_batch==num_mini_batches*0.95:\n            decay_rate=0.00001\n        print(f\"BATCH : {mini_batch+1}\",)\n        for i in range(number_of_iter):\n            \n            \n            A3,cache_params=forward_propogation(mini_X,params,num_of_layers)\n            if i%10==0:\n                print(f\"Training : {i+1}\",end=\" \")\n                my_cost=cost(A3,mini_Y)\n                print(\"COST : \" + str(my_cost) +\" Change : \"+ str(my_cost-mini_cost_tracker[-1]),end=\" \")\n                show_accuracy(mini_Y,A3)    \n            \n            mini_cost_tracker.append(my_cost)\n\n            params = back_propogation(A3,mini_Y,mini_X,cache_params,params,num_of_layers,decay_rate)\n            \n            if i==number_of_iter*0.8:\n                decay_rate=0.0008\n                \n            if i==number_of_iter*0.8:\n                decay_rate=0.0001\n        \n        cost_tracker.append((mini_batch,np.average(mini_cost_tracker)))  \n    predictions,cache_params=forward_propogation(normalize(X),params,num_of_layers)\n    my_cost=cost(predictions,Y)\n    print(\"Final COST : \" + str(my_cost),end=\" \" )\n    show_accuracy(Y,predictions)\n    return params,cost_tracker","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5a56d67-f972-4b1f-9c44-71c8366691a3","_cell_guid":"f2a616ac-4f51-4dc0-a0f0-902c131b243e","trusted":true},"cell_type":"code","source":"np.seterr(all='raise')\nmy_x,my_y= cleanXY(train_data,train_labels)\nmy_x_norm= normalize(my_x)\n\n%time params,cost_tracker=train(my_x,my_y,32,40,[128,64])\n\nprint(\"DONE\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf446d29-a0a0-49be-b15b-06537aa5a66e","_cell_guid":"b26a63d6-2cee-45f5-8f4e-85b2027fa71e","trusted":true},"cell_type":"code","source":"plt.plot([x for x,y in cost_tracker],[y for x,y in cost_tracker])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0384abf-e7f9-46b8-a517-b0c11c0434a6","_cell_guid":"44efa26b-2e2a-43c1-a619-cabf237ee74d","trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"i=28\n\npredictions,cache_params=forward_propogation(my_x_norm[i][0].reshape(1,-1),params,3)\nprint(f\"Number is {np.argmax(my_y[i])} \")\n# print(predictions)\n\nprint(\"Prediction : \" +str(np.argmax(predictions)) )\n# print(np.argmax(predictions,axis=1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7237f48e-d75a-48ba-b0cf-51180969d209","_cell_guid":"84cbf73c-9789-44fa-aa3b-b62b8c1f4f6d","trusted":true},"cell_type":"code","source":"my_test_x,_=cleanXY(test_data)\nmy_norm_test_x=normalize(my_test_x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a061243-0d37-4a03-a3dd-2c13a52b7bf7","_cell_guid":"59c306cc-ab89-4b04-852f-d72b944924f7","trusted":true},"cell_type":"code","source":"predictions,cache_params=forward_propogation(my_norm_test_x,params,3)\ntemp=(pd.DataFrame(np.argmax(predictions,axis=1)))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88b49156-ab66-4c2e-9f21-e3b5ac3be3c2","_cell_guid":"5eac6fc5-168a-49ae-acd7-ea9d474299a2","trusted":true},"cell_type":"code","source":"soln=pd.DataFrame(temp)\nsoln.index+=1\ndisplay(soln)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cda16247-921c-4ec1-813d-a8a7058f84f1","_cell_guid":"80030d48-5fd8-424c-bdb8-57924da6d7d6","trusted":true},"cell_type":"code","source":"x=soln.to_csv(\"/kaggle/working/submission.csv\",header=[\"Label\"],index=True,index_label=[\"ImageId\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"157c32f6-c32d-4ad9-9212-33d6a4074950","_cell_guid":"dddf390e-56ed-4675-b37b-dc83336bdc67","trusted":true},"cell_type":"code","source":"weights.x=np.save(\"/kaggle/working/weights.npy\",params)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ac8b570-4709-4252-947a-897925de82f9","_cell_guid":"a99c5f8d-3c7f-4aa3-9831-6a9f58098b3b","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}