{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6459aaf1ba1cb3b6c1a54ee303e0e1969778ab12","_cell_guid":"3381bfd2-e296-4e4e-8e24-7120dd73d4be"},"cell_type":"markdown","source":"<font color = \"indigo\"><b>Let us start by loading all the relevant libraries and modules that we need for our code</font></b>"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport cv2\nimport sys\nimport random\nimport seaborn as sns\nfrom sklearn import metrics\nfrom sklearn.cross_validation import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"d8adf60c8e63953c1a990b54cbf630cc8012632b","_cell_guid":"b2893b46-0920-4cea-be23-f7486ed5b294","trusted":true},"cell_type":"code","source":"Train = []\nTrain_Samples = []\nTrain_Labels = []\ntrain_path = '../input/train'\ntest_path = '../input/test'\n\ndog_count = 0\ncat_count = 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"271b74c83f1f32900725b3efcffe992e70745c75","_cell_guid":"c19eb668-8261-4822-a355-7438dc0c7c3e","trusted":true},"cell_type":"code","source":"Train = [img for img in os.listdir(train_path)]\n\nprint (Train[0:5])","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"9627ef44826edd4c069a7fa8525daa9da10b915d","_cell_guid":"799d2073-30ac-454f-94b8-30b290756428","trusted":true},"cell_type":"code","source":"### Generate ~ 2000 pics for dogs and cats | Cats = 0 & Dogs = 1 for numeric representation\n\ntraining_set_per_example = 1000\n\nfor i,pic in enumerate(Train):\n    if Train[i][0:3].lower() == 'cat' and cat_count < training_set_per_example:\n        Train_Samples.append(cv2.resize(cv2.imread(train_path + '/' + Train[i]),(256,256)).flatten())    ### Cats are denoted as Zeros\n        Train_Labels.append(0)\n        cat_count = cat_count + 1 \n        # print(\"Cat Image number \" , pic , \" added as sample number \" , i , \" to the training set \")\n    elif Train[i][0:3].lower() == 'dog' and dog_count < training_set_per_example:\n        Train_Samples.append(cv2.resize(cv2.imread(train_path + '/' + Train[i]),(256,256)).flatten())   ### Dogs are denoted as 1s\n        Train_Labels.append(1)\n        dog_count = dog_count + 1 ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"24c9044fc140b9df1e090a6f2f160e2263adde13","_cell_guid":"576dfafe-9c87-4208-82d9-1a0d01ba61d5","trusted":true},"cell_type":"code","source":"#Split Data into Train & Dev\ntrain_x, dev_x, train_y, dev_y = train_test_split(Train_Samples,Train_Labels,train_size = 0.8, random_state = 10)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"37da28b1ab88cf07e5e17b225e4eed2f41cea7e3","_cell_guid":"7685d25f-23f1-4996-8bc8-2facc6b2ce5a","trusted":true},"cell_type":"code","source":"## Convert Lists to arrays to make calculations for Neural Network easy \nX_Train_temp = np.array(train_x)\nY_Train_temp = np.array(train_y)\nDev_X_temp = np.array(dev_x)\nDev_Y_temp = np.array(dev_y)\n\n## Reshape all arrays to ensure all the dimensions are correct\nX_Train = X_Train_temp.reshape(X_Train_temp.shape[1],X_Train_temp.shape[0]) / 255\nDev_X = Dev_X_temp.reshape(Dev_X_temp.shape[1],Dev_X_temp.shape[0]) / 255\nY_Train = Y_Train_temp.reshape(Y_Train_temp.shape[0],1) \nDev_Y = Dev_Y_temp.reshape(Dev_Y_temp.shape[0],1)\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"2e37b95c871deab9f25c4fa15fc34957a039445b","_cell_guid":"b25aa9a4-3ac7-499e-9025-530a58cb0193","trusted":true},"cell_type":"code","source":"Dog_Images = Y_Train[Y_Train == 1]\nCat_Images = Y_Train[Y_Train == 0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ee8579ab4abb66ce43697865382bcb7c46cd9fa","_cell_guid":"79303524-7a89-452d-9a3a-681860ae27e1","trusted":true},"cell_type":"code","source":"## Print all array shapes to cross verify their size and shape \nprint(\"Training Input Shape:\" ,X_Train.shape)\nprint(\"Training Output Shape:\" , Y_Train.shape)\nprint(\"Validation Input shape:\" ,Dev_X.shape)\nprint(\"Validation Output Shape:\" , Dev_Y.shape)\nprint(\" \")\nprint(\"No. of Training examples is \", X_Train.shape[1])\nprint(\"No. of validation examples is \", Dev_X.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90e992026e3ac0a2d41b83265961fee42898c507","_cell_guid":"1d55ecf2-2aab-454f-8c91-cddee3d8d464"},"cell_type":"markdown","source":"<font><b>We will now start writing all the helper functions - which we will call in our main Neural Network block. All the individual functions are designed to do a very specific task starting from initialization of variables to forward propagation and then backward propagation logic</font></b>"},{"metadata":{"collapsed":true,"_uuid":"6c6a8ce254dfa5fd174020fc37da3e525167badf","_cell_guid":"475c65a2-aa0a-472a-af21-eac2d0bb6bc9","trusted":true},"cell_type":"code","source":"# Here, we need to initialize W and b for the number of layers that are passed as parameter \n# Dimension of W(l) = [n(l), n(l-1)]\n# Dimension of b(l) = [n(l),1]\n# where n(l) is the number of neurons in lth layer and n(l-1) is the number of neurons in (l-1)th layer - this information resides in layers_dims array\n# For l=1, n(l-1) will be the shape of individual input training examples i.e the shape of individual flattened image\n\ndef initialize_parameters(layers_dims,seed):\n    np.random.seed(seed)\n    parameters = {}\n    no_of_layers = len(layers_dims) - 1 \n    for l in range(1,(no_of_layers+1)):\n        parameters['W'+str(l)] = np.random.randn(layers_dims[l],layers_dims[l-1]) * np.sqrt(2/layers_dims[l-1])\n        parameters['b'+str(l)] = np.zeros((layers_dims[l],1), dtype=np.float64)\n        \n        assert(parameters['W' + str(l)].shape == (layers_dims[l], layers_dims[l-1]))\n        assert(parameters['b' + str(l)].shape == (layers_dims[l], 1))\n\n    return parameters","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"5c6c01a1a7a70ae81a86bde276750395cd609550","_cell_guid":"d868f96a-5759-4a61-8d05-e441347b658c","trusted":true},"cell_type":"code","source":"# This block will output tr output that will be passed to an activation layer\ndef linear_forward(X,W,b):\n    Z = np.dot(W,X) + b\n    \n    assert(Z.shape == (W.shape[0], X.shape[1]))\n    \n    return Z","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"0587ac50c0b3115811497e44911fc0ba6aac8a28","_cell_guid":"918ddca5-eb48-41cd-9240-81c19fb26484","trusted":true},"cell_type":"code","source":"# This block will convert linear_forward output into a non-linear activation output using various activation functions\n# This block writes code for 2 activations - \"sigmoid' and 'leaky relu' with alpha value of 0.01 \ndef non_linear_forward(Z,activation):\n    if activation == 'sigmoid':\n        A = 1 /(1 + np.exp(-Z))\n        \n    elif activation == 'relu':\n        t = np.ones(Z.shape)\n        t[Z<=0] = 0.01\n        A = Z * t\n        \n    return A\n    ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"e1a74e1e847059449ffe7c8de614efe71cbf8afb","_cell_guid":"6e9cb224-dbce-4845-95b9-8780104713f9","trusted":true},"cell_type":"code","source":"# This block will write code for forward propagation and store W,b and Z values corresponding to every layer in a cache\n\ndef forward_prop(X_Train,parameters):\n    saved = {}\n    caches = []\n    Z = []\n    A_prev = X_Train\n    for l in range(1,(no_of_layers+1)):\n        Z = linear_forward(A_prev,parameters['W'+str(l)],parameters['b'+str(l)])\n        A = non_linear_forward(Z,activations[l-1])\n        saved['Z'+str(l)] = Z \n        cache = A_prev,parameters['W'+str(l)],parameters['b'+str(l)]\n        caches.append(cache)\n        A_prev = A \n    # Save last output of activation to initiate backpropagation logic\n    AL = A \n    return AL,saved,caches","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"400478e3c2fc102eb14141b3a3cc7bfc52537f70","_cell_guid":"7a638741-3738-4f6a-bbb4-699e2c1c298d","trusted":true},"cell_type":"code","source":"# Compute cost \ndef compute_cost(AL,Y_Train):\n    m = Y_Train.shape[1]\n    cost = (-1/m) * (np.sum(np.multiply(Y_Train,np.log(AL)) + np.multiply((1-Y_Train),np.log((1-AL)))))\n    cost = np.squeeze(cost)\n    \n    return cost","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"25562e6217012efb097321abfe878feaf0c74c98","_cell_guid":"d2868930-ead2-4187-9789-42d72b433fb0","trusted":true},"cell_type":"code","source":"# Initialize backpropagation using AL \ndef initialize_backprop(AL,Y_Train):\n    dAL = - (np.divide(Y_Train, AL) - np.divide(1 - Y_Train, 1 - AL)) \n    return dAL","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"e1e6938a97ee662b973b68b58dce3bc5db01d895","_cell_guid":"a56e448a-4560-45ed-8664-7cf19b04c25a","trusted":true},"cell_type":"code","source":"# Backward propagation for Activation function - Returns gradient on activation functions\ndef backward_prop_activation(dA,saved,activation,layer):\n    if activation == 'sigmoid':\n        Z = saved['Z'+str(layer)]\n        s = 1/(1 + np.exp(-Z))\n        dZ = dA * s * (1-s)\n        \n    elif activation == 'relu':\n        Z = saved['Z'+str(layer)]\n        dZ = np.array(Z,copy=True)\n        dZ[Z>0] = 1\n        dZ[Z<0] = 0.01\n        dZ = dA*dZ\n       \n    assert(dZ.shape == Z.shape)\n\n    return dZ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"5a844ee85e52522daf9282fd50c344f322f1e0aa","_cell_guid":"616e4979-499d-4ac2-a19b-386319d4ca48","trusted":true},"cell_type":"code","source":"# Backward propagation for the linear part(s)\ndef backward_prop_linear(dZ,caches,l):\n    A_prev,W,b = caches[l-1]\n    \n    m = A_prev.shape[1]\n    dW = (1/m) * np.dot(dZ,A_prev.T)\n    db = (1/m) * np.sum(dZ,axis=1,keepdims=True)\n    dA_prev = np.dot(W.T,dZ)\n          \n    assert (dA_prev.shape == A_prev.shape)\n    assert (dW.shape == W.shape)\n    assert (db.shape == b.shape)\n    \n    return dA_prev,dW,db","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"d9ee6514b0ae60e5a39bde28fe22c6c166700056","_cell_guid":"ec0af5d4-55d2-45b2-b102-0955ea0abcf9","trusted":true},"cell_type":"code","source":"# Store Gradients\ndef store_gradients(dA_prev,dW,db,layer):\n    grads = {}\n    grads[\"dA\" + str(layer)] = dA_prev\n    grads[\"dW\" + str(layer)] = dW\n    grads[\"db\" + str(layer)] = db\n    \n    return grads\n        ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"2b05853819f472a5fe083629e64d28c7ed89410c","_cell_guid":"9d4975d6-4e37-4d79-b192-3615f484e18b","trusted":true},"cell_type":"code","source":"# Update Gradients \ndef update_gradients(parameters,grads,learning_rate,layer):\n    parameters[\"W\"+str(layer)] = parameters[\"W\"+str(layer)] - (learning_rate * grads[\"dW\"+str(layer)])\n    parameters[\"b\"+str(layer)] = parameters[\"b\"+str(layer)] - (learning_rate * grads[\"db\"+str(layer)])\n    \n    return parameters","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"95925e68080bc3addf293ea7031a3008b243db6a","_cell_guid":"1fc43712-bf3f-4a4f-afa4-e47da7b7f032","trusted":true},"cell_type":"code","source":"# Define Hyperparameters and Layers\n## layers_dims contains information about [input neurons, hidden layer 1 neurons,hidden layer 2 neurons .... hidden layer n neurons, \n## output layer neurons]\nlayers_dims = [X_Train.shape[0],1000,1000,1]\nactivations = ['relu','relu','sigmoid']\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"351d54414062fe6fd70af07d041b5e0d02e56e0f","_cell_guid":"e7af3e89-ae62-42fe-84d4-55ffcd345af4"},"cell_type":"markdown","source":"<font color = \"blue\" ><b>The below block calls all the helper functions coded above to form a complete neural network architechture. The intent is to run this block multiple times (epocs) to keep on updating parameters (W & b) to minimize the cost</font></b>"},{"metadata":{"scrolled":true,"_uuid":"f6eb06f4a41e893272373aa8e389a62f1ee7bd0c","_cell_guid":"ef3ed41d-226d-4ff6-8d81-bf917d0acab3","trusted":true},"cell_type":"code","source":"# Build a full fledged neural network layer with forward and backward propagation logic\n# Activation functions to be used are picked from \"activations\" array.\n\n\ncosts = []\nno_of_layers = len(layers_dims) - 1\nepochs = 200\nseed = 192\nlearning_rate = 0.01\n\nparameters = initialize_parameters(layers_dims,seed)\n\nfor i in range(epochs):\n    AL, saved,caches = forward_prop(X_Train,parameters)\n    cost = compute_cost(AL,Y_Train)\n    costs = np.append(costs,cost)    \n   \n    print(\"Cost at epoch \", i , \" is \" , cost)\n    Y_Train = Y_Train.reshape(AL.shape)\n    dAL = initialize_backprop(AL,Y_Train)\n    \n    dA = dAL \n\n    for l in reversed(range(1,(no_of_layers+1))):\n        dZ = backward_prop_activation(dA,saved,activations[l-1],l)\n        dA_prev,dW,db = backward_prop_linear(dZ,caches,l)\n        grads = store_gradients(dA_prev,dW,db,l)\n        parameters = update_gradients(parameters,grads,learning_rate,l)\n        dA = dA_prev\n    \n%matplotlib inline\nplt.plot(costs)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05d183ef621819c4f94a08effb7a0b9a8a43c837","_cell_guid":"64e3c85c-b6ab-4884-9255-4dadd89267e5","trusted":true},"cell_type":"code","source":"print(AL)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"collapsed":true,"_uuid":"219e761d7d5db8857da9a6c0c6e57af3b4f5716c","_cell_guid":"961fbf94-2ae9-4d8a-88e2-c804991678f0","trusted":true},"cell_type":"code","source":"temp = []\nfor x in AL.flatten():\n    if x >=0.5:\n        temp.append(1)\n    else:\n        temp.append(0)\n        ","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"collapsed":true,"_uuid":"2eff503deb8597a1ebe01d99bb22214dbc9321a2","_cell_guid":"9515a43c-c106-406b-b358-e2116b060b95","trusted":true},"cell_type":"code","source":"cf = metrics.confusion_matrix(Y_Train.flatten(),temp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0039fe24f6684b8e981de6e92f0f4013d93cf13c","_cell_guid":"25d474e9-1f2a-4025-aaa1-31610c95e45c","trusted":true},"cell_type":"code","source":"print(cf)\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"6a6336a1fb76ccca6af20c5a9b27c742f6185aad","_cell_guid":"876196ce-d539-4d25-96d1-c8d224612777","trusted":true},"cell_type":"code","source":"acc = metrics.accuracy_score(Y_Train.flatten(),temp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad4ace657f9d85a1d5bc42196f072aafa5639555","_cell_guid":"3ceecd98-d951-4a9e-99e4-80509702a998","trusted":true},"cell_type":"code","source":"acc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e31ab82053c774ad530b489fcdb4dfebb45f002a","_cell_guid":"717f66e2-68aa-4863-99bf-b3dec43639b0","trusted":true},"cell_type":"code","source":"metrics.f1_score(Y_Train.flatten(),temp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d8b3b7e043550ca80ef0a9d9a5016280c267bd5","_cell_guid":"54ad4fd1-6153-4a7c-84c0-41c0ab75ad07","trusted":true},"cell_type":"code","source":"# Let's randomly check mislabeled images\nmisclass = []\ncorrect = []\nfor x,predictions in enumerate(temp):\n    if temp[x] == Y_Train.flatten()[x]:\n        correct.append(x)\n    else:\n        misclass.append(x)\n\nprint(\"Misclassified indexes :\")\nprint(misclass[0:50])\n\nprint(\"Correct indexes :\")\nprint(correct[0:50])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"145d58c3f759de0dfd901ceb052054fa177140eb","_cell_guid":"38f5bf4a-c083-4644-b7ec-11745a231dac","trusted":true},"cell_type":"code","source":"%matplotlib inline\nidx = 55\nprint(\"Model predicted CAT for this image \" if temp[idx] == 0 else \"Model predicted DOG for this image \")\nplt.imshow(X_Train_temp[idx].reshape(256,256,3))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_uuid":"e9a8eb3cb80b4865a554b922e92e03f001efe617","_cell_guid":"961208e8-90e2-42bb-9ed2-b3e87e0e9057","trusted":true},"cell_type":"code","source":"# Let's Check CV Accuracy\n\nAL_D,saved_D,caches_D = forward_prop(Dev_X,parameters)\n\ntemp_D = []\nfor x in AL_D.flatten():\n    if x >=0.5:\n        temp_D.append(1)\n    else:\n        temp_D.append(0)\n        \ncf_D = metrics.confusion_matrix(Dev_Y.flatten(),temp_D)\nprint(cf_D)\nacc_D = metrics.accuracy_score(Dev_Y.flatten(),temp_D)\nprint(acc_D)\nmetrics.f1_score(Dev_Y.flatten(),temp_D)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}