{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport math\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Data preprocessing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"dt=pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndtest=pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dt.head)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y=dt.iloc[:,0].values\nX=dt.iloc[:,1:].values\ndtest__=dtest.iloc[:,:].values\nprint(\"the number of samples \"+ str(X.shape[0]))\nprint(\"the shape of trainig data \"+ str(X.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Encoding the target \ny_train_=np.zeros((10,42000))\nfor col in range (42000):\n    val=Y[col]\n    for row in range (10):\n        if (val==row):\n            y_train_[val,col]=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reshaping data \nx_train_flatten=X.reshape(X.shape[0],-1).T\ndtest_flatten=dtest__.reshape(dtest__.shape[0],-1).T\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Standardize data \nX_train=x_train_flatten/255\nX_test=dtest_flatten/255","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Modeling**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialize_parameters_deep(layer_dims):\n    np.random.seed(1)\n    parameters = {}\n    L = len(layer_dims)\n    for l in range(1, L):\n        parameters['W' + str(l)] =np.random.randn(layer_dims[l],layer_dims[l-1])*0.01\n        parameters['b' + str(l)] = np.zeros((layer_dims[l],1))\n    return parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def linear_forward(A, W, b):     \n    Z = np.dot(W,A)+b\n    assert(Z.shape == (W.shape[0], A.shape[1]))\n    cache = (A, W, b)\n    \n    return Z, cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def relu(Z):\n    A = np.maximum(0,Z)    \n    cache = Z \n    return A, cache\n\ndef softmax(Z):\n    A= np.exp(Z)/np.sum(np.exp(Z),axis=0)  \n    cache=Z\n    return A,cache   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def linear_activation_forward(A_prev, W, b, activation):\n    Z,linear_cache = linear_forward(A_prev, W, b)\n    if activation == \"relu\": \n        A, activation_cache = relu(Z)\n    elif activation == \"softmax\":\n             \n             A, activation_cache = softmax(Z) \n            \n            \n    assert (A.shape == (W.shape[0], A_prev.shape[1]))\n    cache = (linear_cache, activation_cache)\n\n\n    return A, cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def L_model_forward(X, parameters):\n    caches = []\n    A = X\n    L = len(parameters) // 2   \n    for l in range(1, L):\n        A_prev = A \n        A, cache = linear_activation_forward(A_prev, parameters['W' + str(l)],parameters['b' + str(l)], activation = \"relu\")\n        caches.append(cache)\n  \n    AL, cache = linear_activation_forward(A, parameters['W' + str(L)],parameters['b' + str(L)], activation = \"softmax\")\n    caches.append(cache)\n    \n    #assert(AL.shape == (1,X.shape[1]))\n            \n    return AL, caches","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_cost(AL, Y):\n    m = Y.shape[1]\n    #print(AL.shape)\n    Y= np.reshape(Y,(Y.shape[0],Y.shape[1]))\n    cost = (-1/m)*(np.multiply(Y,np.log(AL)))\n    \n    cost = np.squeeze(cost) \n    #assert(cost.shape == ())\n    \n    return cost\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def linear_backward(dZ, cache):\n    A_prev, W, b = cache\n    m = A_prev.shape[1]\n    dW = 1/m*np.dot(dZ,A_prev.T)\n    db = 1/m*np.sum(dZ,axis=1,keepdims=True)\n    dA_prev = np.dot(W.T,dZ)\n    \n    assert (dA_prev.shape == A_prev.shape)\n    assert (dW.shape == W.shape)\n    assert (db.shape == b.shape)\n    \n    return dA_prev, dW, db","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def relu_backward(dA, cache):\n    Z = cache\n    dZ = np.array(dA, copy=True)\n    dZ[Z <= 0] = 0\n    assert (dZ.shape == Z.shape)\n    return dZ\ndef softmax_backward(Z,cache):\n    Z=cache\n    length=10  \n    dZ=np.zeros((29400,10))\n    Z=np.transpose(Z)\n    for row in range (0,29400):\n            den=(np.sum(np.exp(Z[row,:])))*(np.sum(np.exp(Z[row,:])))\n            for col in range (0,10):\n                sums=0\n                for j in range (0,10):\n                    if (j!=col):\n                        sums=sums+(math.exp(Z[row,j]))\n                \n                dZ[row,col]=(math.exp(Z[row,col])*sums)/den           \n    dZ=np.transpose(dZ)\n    Z=np.transpose(Z)\n\n    assert (dZ.shape == Z.shape)\n    return dZ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def relu_prime(Z):\n    Z[Z<=0] = 0\n    Z[Z>0] = 1\n    return Z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef linear_activation_backward(dA, cache, activation):\n   \n    linear_cache, activation_cache = cache\n    \n    if activation == \"relu\":\n        dZ = dA*relu_prime(activation_cache)\n        dA_prev, dW, db = linear_backward(dZ,linear_cache)\n        \n    elif activation == \"softmax\":\n        dZ = dA\n        dA_prev, dW, db =linear_backward(dZ,linear_cache)\n        \n    \n    return dA_prev, dW, db","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def L_model_backward(AL, Y, caches):\n    grads = {}\n    L = len(caches) # the number of layers\n    m = AL.shape[1]\n    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n    \n    # Initializing the backpropagation\n    \n    dAL = AL-Y\n   \n    current_cache = caches[L-1]\n    grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(dAL, caches[L-1], \"softmax\")\n   \n    \n    for l in reversed(range(L-1)):\n       \n        current_cache = caches[l]\n        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\" + str(l + 1)], current_cache, \"relu\")\n        grads[\"dA\" + str(l)] =dA_prev_temp\n        grads[\"dW\" + str(l + 1)] =dW_temp\n        grads[\"db\" + str(l + 1)] =db_temp\n        \n\n    return grads","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_parameters(parameters, grads, learning_rate):\n    \n    L = len(parameters) // 2 # number of layers in the neural network\n\n    for l in range(L):\n        parameters[\"W\" + str(l+1)] =parameters[\"W\" + str(l+1)]-learning_rate*(grads[\"dW\" + str(l+1)])\n        parameters[\"b\" + str(l+1)] =parameters[\"b\" + str(l+1)]-learning_rate*(grads[\"db\" + str(l+1)])\n    return parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mini_batch_gradient_descent(X,Y,layer_dims,mini_batch_size,epochs, learning_rate=0.0075):\n  ############## Creating Mini Batches ##########################\n    np.random.seed(1)\n    m = X.shape[1]\n    mini_batches = []\n    \n    #shuffling the data\n    permutation = list(np.random.permutation(X.shape[1]))\n    X_shuffled = X[:, permutation]\n    Y_shuffled = Y[:, permutation]\n    num_of_complete_batches = m // mini_batch_size\n    for i in range(num_of_complete_batches):\n        mini_batch_X = X_shuffled[:,i*mini_batch_size:(i+1)*mini_batch_size]\n        mini_batch_Y = Y_shuffled[:,i*mini_batch_size:(i+1)*mini_batch_size]\n        mini_batch = (mini_batch_X,mini_batch_Y)\n        mini_batches.append(mini_batch)\n        \n    #if there is incomplete batch\n    if m % mini_batch_size != 0:\n       \n        mini_batch_X=X_shuffled[:,num_of_complete_batches*mini_batch_size:num_of_complete_batches*mini_batch_size + (m - mini_batch_size*num_of_complete_batches)]\n        mini_batch_Y=Y_shuffled[:,num_of_complete_batches*mini_batch_size:num_of_complete_batches*mini_batch_size + (m - mini_batch_size*num_of_complete_batches)]\n        mini_batch = (mini_batch_X,mini_batch_Y)\n        mini_batches.append(mini_batch)\n    \n    ############## Training Model Using Mini Batches ##########################\n    #parameters_initialize\n    costs=[]\n    parameters = initialize_parameters_deep(layer_dims)\n    for j in range(epochs):\n        for mini_batch in mini_batches:\n            x_batch,y_batch=mini_batch\n            A_Last,caches = L_model_forward(x_batch,parameters)\n            cost = np.sum(compute_cost(A_Last, y_batch))/A_Last.shape[0]\n            grads = L_model_backward(A_Last, y_batch , caches)\n            parameters = update_parameters(parameters, grads, learning_rate)\n        if j%2 == 0:\n            print(f'Iteration {j} : {cost}')\n            costs.append(cost)\n    return parameters,costs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"layers_dims = [784,100,40,10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameters,costs =  mini_batch_gradient_descent(X_train,y_train_,layers_dims,mini_batch_size=9,epochs =60,learning_rate = 0.006 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(costs)\nplt.ylabel('Cost')\nplt.xlabel('Iterations per 2 epochs')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Predection "},{"metadata":{"trusted":true},"cell_type":"code","source":"last_test,caches=L_model_forward(X_test,parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=np.argmax(last_test,axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\ndf_submission['Label'] = pred\ndf_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.to_csv('output.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}