{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\nfrom matplotlib import pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nX_test  = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\ndisplay(data)\n\ntrain_data = data[: (int((len(data))*0.8))]\ncv_data = data[(int((len(data))*0.8)) :] \n\nX_train = np.array(train_data.drop(\"label\" , axis = 1).copy()).T\nY_train = np.array([train_data['label'].copy()])\n\nX_cv = np.array(cv_data.drop(\"label\" , axis = 1).copy()).T\nY_cv = np.array([cv_data['label'].copy()])\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = np.array(X_train)\nY_train = np.array(Y_train)\n\nX_cv = np.array(X_cv)\nY_cv = np.array(Y_cv)\n\nX_test  = np.array(X_test).T\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train.shape,Y_train.shape)\nprint(X_cv.shape,Y_cv.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shape = (Y_train.max()+1, Y_train.shape[1]) #Generating one_hot_vector so that later we can apply multi class classification\none_hot = np.zeros(shape)\nrows= np.arange(Y_train.shape[1])\none_hot[Y_train , rows]=1\nY_train = one_hot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shape = (Y_cv.max()+1, Y_cv.shape[1]) #Generating one_hot_vector so that later we can apply multi class classification\none_hot = np.zeros(shape)\nrows= np.arange(Y_cv.shape[1])\none_hot[Y_cv , rows]=1\nY_cv = one_hot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(Y_train.shape,Y_cv.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = np.array(X_train/255)\nX_cv = np.array(X_cv/255)\n\nX_test = np.array(X_test/255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_X=X_train.shape[0]\nm=X_train.shape[1]\n\ndef softmax(A):               #Softmax gives a good classification as it standardizes the output values.\n    A = A - np.max(A)\n    return np.exp(A) / np.sum(np.exp(A),axis=0) \n\nnums = np.array([4, 5, 6])\nprint(softmax(nums))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#def random_mini_batches(X,Y,mini_bactch_size=64):\nm = X_train.shape[1] \nmini_batches = []\n\npermutation = list(np.random.permutation(m))\nshuffled_X = X_train[:,permutation]\nshuffled_Y = Y_train[:,permutation]\n\nprint(shuffled_X.shape)\nprint(shuffled_Y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_mini_batches(X,Y,mini_batch_size=64):\n  m = X.shape[1] \n  mini_batches = []\n\n  permutation = list(np.random.permutation(m))\n  shuffled_X = X [:,permutation]\n  shuffled_Y = Y [:,permutation]\n\n  num_comp_batchs = math.floor (m/mini_batch_size)\n\n  for i in range (num_comp_batchs):\n    mini_batch_X = shuffled_X [:, i*mini_batch_size : (i+1)*mini_batch_size ]\n    mini_batch_Y = shuffled_Y [: ,i*mini_batch_size : (i+1)*mini_batch_size ]\n\n    mini_batch = ( mini_batch_X, mini_batch_Y )\n    mini_batches.append(mini_batch)\n\n  if m % mini_batch_size != 0:\n    end = m - mini_batch_size*math.floor(m/mini_batch_size)\n    mini_batch_X = shuffled_X [:, num_comp_batchs*mini_batch_size : ]\n    mini_batch_Y = shuffled_Y [:, num_comp_batchs*mini_batch_size : ]\n\n    mini_batch =( mini_batch_X, mini_batch_Y )\n    mini_batches.append(mini_batch)\n\n  return mini_batches\n\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#initialize_parameters\nlayer_dims=[784,1000,500,250,100,50,10]\n\nparameters={}\nL=len(layer_dims)\nfor l in range(1,L):\n  parameters[\"W\"+str(l)]= np.random.randn(layer_dims[l],layer_dims[l-1])*np.sqrt(2/layer_dims[l-1])\n\n  parameters[\"b\"+str(l)]= np.zeros((layer_dims[l],1))*0.01\n\n\ndef DNN_model(X,Y,parameters,layer_dims,num_iterations,learning_rate):\n\n  m=X.shape[1]\n  Y_prediction=np.zeros((1,m))\n  forward_cache={}\n  \n  \n  L=len(layer_dims)\n  grads={}\n  costs=[]\n  cost_1=1000\n  cost_2=1000\n  for i in range(0,num_iterations):\n\n        \n    mini_batches = random_mini_batches(X,Y,mini_batch_size=32)\n\n    for mini_batch in mini_batches:\n      (mini_batch_X,mini_batch_Y) = mini_batch\n      forward_cache[\"A0\"]=np.copy(mini_batch_X)\n    \n      A=np.copy(mini_batch_X)\n      for l in range(1,L-1):\n      \n        A_prev=np.copy(A)\n      \n        forward_cache[\"Z\"+str(l)]=np.dot(parameters[\"W\"+str(l)],A_prev)+parameters[\"b\"+str(l)]\n        A=np.maximum(0,forward_cache[\"Z\"+str(l)])\n      \n        forward_cache[\"A\"+str(l)]=np.copy(A)\n    #last_layer\n      W=parameters[\"W\"+str(L-1)]\n      b=parameters[\"b\"+str(L-1)]\n      forward_cache[\"Z\"+str(L-1)]=np.dot(W,A)+b\n      AL_train=  softmax(forward_cache[\"Z\"+str(L-1)])\n#       print(AL_train.shape)\n    #compute_cost\n      m=mini_batch_Y.shape[1]\n#       print(mini_batch_Y.shape,AL_train.shape)\n      cost_1 = -np.mean(mini_batch_Y * np.log(AL_train + 1e-8))\n\n    #print(np.sum(W))\n    #print(np.sum(b))\n      print(\"After {} epoch cost is {}\".format(i,cost_1))\n\n      costs.append(float(cost_1))\n    #backward_prop\n      grads[\"dZ\"+str(L-1)]=AL_train-mini_batch_Y\n      grads[\"dW\"+str(L-1)]=(1/m)*(np.dot(grads[\"dZ\"+str(L-1)],forward_cache[\"A\"+str(L-2)].T))\n      grads[\"db\"+str(L-1)]=(1/m)*(np.sum(grads[\"dZ\"+str(L-1)],axis=1,keepdims=True))\n      for l in reversed(range(1,L-1)):\n        grads[\"dZ\"+str(l)]=np.multiply(np.dot(parameters[\"W\"+str(l+1)].T,grads[\"dZ\"+str(l+1)]),np.int64(forward_cache[\"A\"+str(l)]>0))\n        grads[\"dW\"+str(l)]=(1/m)*(np.dot(grads[\"dZ\"+str(l)],(forward_cache[\"A\"+str(l-1)]).T))\n        grads[\"db\"+str(l)]=(1/m)*(np.sum(grads[\"dZ\"+str(l)],axis=1,keepdims=True))\n      for l in range(1, L):\n        parameters[\"W\" + str(l)] = parameters[\"W\" + str(l)] - learning_rate * grads[\"dW\" + str(l)]\n        parameters[\"b\" + str(l)] = parameters[\"b\" + str(l)] - learning_rate * grads[\"db\" + str(l)]\n     \n  return parameters,costs,AL_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameters,costs,AL_train= DNN_model(X_train,Y_train,parameters,layer_dims,num_iterations=5,learning_rate=0.1)\nnum_iterations=5\n\nplt.plot(list(range(len(costs))),list(costs))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef DNN_model_train(X,Y,parameters):\n\n  m=X.shape[1]\n  Y_prediction=np.zeros((1,m))\n  forward_cache={}\n  \n  forward_cache[\"A0\"]=np.copy(X)\n  L=len(layer_dims)\n  grads={}\n  costs=[]\n  cost_1=1000\n  cost_2=1000\n  for i in range(0,num_iterations):\n    \n    A=np.copy(X)\n    for l in range(1,L-1):\n#       print(i,l)\n      A_prev=np.copy(A)\n      \n      forward_cache[\"Z\"+str(l)]=np.dot(parameters[\"W\"+str(l)],A_prev)+parameters[\"b\"+str(l)]\n      A=np.maximum(0,forward_cache[\"Z\"+str(l)])\n      \n      forward_cache[\"A\"+str(l)]=np.copy(A)\n    #last_layer\n    W=parameters[\"W\"+str(L-1)]\n    b=parameters[\"b\"+str(L-1)]\n    forward_cache[\"Z\"+str(L-1)]=np.dot(W,A)+b\n    AL_train=  softmax(forward_cache[\"Z\"+str(L-1)])\n     \n  return parameters,AL_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for train set\nparameters,AL_train= DNN_model_train(X_train,Y_train,parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  for i in range(AL_train.shape[1]):\n    \n    a = np.max(AL_train[:,i])\n    AL_train[:,i] = (AL_train[:,i]==a)\n  fai = np.absolute(AL_train - Y_train)\n  count = 0\n  for i in range(0,fai.shape[1]):\n    if np.sum(fai[:,i]) == 0:\n      count +=1\n  print(count)\n  print((count /X_train.shape[1])*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for_cross_validation\nparameters,AL_cv= DNN_model_train(X_cv,Y_cv,parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  for i in range(AL_cv.shape[1]):\n    \n    a = np.max(AL_cv[:,i])\n    AL_cv[:,i] = (AL_cv[:,i]==a)\n  fai = np.absolute(AL_cv - Y_cv)\n  count = 0\n  for i in range(0,fai.shape[1]):\n    if np.sum(fai[:,i]) == 0:\n      count +=1\n  print(count)\n  print((count /X_cv.shape[1])*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef DNN_model_test(X,parameters):\n\n  m=X.shape[1]\n  Y_prediction=np.zeros((1,m))\n  forward_cache={}\n  \n  forward_cache[\"A0\"]=np.copy(X)\n  L=len(layer_dims)\n  grads={}\n  costs=[]\n  cost_1=1000\n  cost_2=1000\n  for i in range(0,num_iterations):\n    \n    A=np.copy(X)\n    for l in range(1,L-1):\n#       print(i,l)\n      A_prev=np.copy(A)\n      \n      forward_cache[\"Z\"+str(l)]=np.dot(parameters[\"W\"+str(l)],A_prev)+parameters[\"b\"+str(l)]\n      A=np.maximum(0,forward_cache[\"Z\"+str(l)])\n      \n      forward_cache[\"A\"+str(l)]=np.copy(A)\n    #last_layer\n    W=parameters[\"W\"+str(L-1)]\n    b=parameters[\"b\"+str(L-1)]\n    forward_cache[\"Z\"+str(L-1)]=np.dot(W,A)+b\n    AL_test=  softmax(forward_cache[\"Z\"+str(L-1)])\n     \n  return AL_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for test set\nAL_test= DNN_model_test(X_test,parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(AL_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_test_pred = np.array([np.argmax(AL_test,axis=0)]).T\nprint(Y_test_pred.shape)\nprint(Y_test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sequence = np.array([np.arange(1,AL_test.shape[1]+1)]).T\nprint(sequence.shape)\n\noutput = np.hstack([sequence, Y_test_pred])\nprint(output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.savetxt(\"newresults.csv\",output,delimiter = \",\" ,fmt = \"%d\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}