{"cells":[{"metadata":{"_cell_guid":"7e5b6bd4-ab43-45dc-825e-03e05a2e10d1","_uuid":"86c3f1d7587cc2bb0d942cb9557444e88da2b6bc","trusted":false,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt # Visualization\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4296aaca-189e-44de-8682-95366007a650","_uuid":"3cc51c0af847a0c2582a5a23f220bd0824c263c7","trusted":false,"collapsed":true},"cell_type":"code","source":"# Load Data\ntrain_raw = pd.read_csv(\"../input/train.csv\")\ntest_raw = pd.read_csv(\"../input/test.csv\")\nprint(\"Data Loaded\")","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"1058620b-9a97-45f8-9d03-8a9c28914414","_uuid":"ea8572f2f305d9afe287f791557938d7f6207ffd","trusted":false},"cell_type":"code","source":"def get_features(raw_data):\n    cols = []\n    # Get data of each row from pixel0 to pixel783 \n    for px in range(784):\n        cols.append(\"pixel\"+str(px))   \n    return raw_data.as_matrix(cols)/255\n\ndef cross_validated(X, n_samples):\n    kf = KFold(n_samples, shuffle = True)\n    result = [group for group in kf.split(X)]\n    return result        ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"633909b5-75bf-47af-acfb-6f13b5e1098a","_uuid":"e356a012e7cca32c2ea030be631158dc9fe9405b","trusted":false},"cell_type":"code","source":"# Deep Implementation \n# Initialize Parameters \ndef init_dnn_parameters(n, activations, epsilons, filter=None):\n    L = len(n)\n    params = {}\n    for l in range(1,L):\n        W = np.random.randn(n[l],n[l-1]) * epsilons[l] \n        # Experiment, multiply filter in case of input layer weights \n        if filter1 is not None and l == 1:\n            W = np.dot(W, filter) \n        b = np.zeros((n[l],1))\n        params[\"W\"+str(l)] = W\n        #print(\" initizlizing W\"+str(l))\n        params[\"b\"+str(l)] = b                        \n        params[\"act\"+str(l)] = activations[l]\n    params[\"n\"] = n\n    return params\n\n# Activation Functions \ndef gdnn(X, activation_function):\n    leak_factor = 1/10000\n    if activation_function == 'tanh':\n        return np.tanh(X)\n    if activation_function == 'lReLU':\n        return ((X > 0) * X) + ((X <= 0)* X * leak_factor)\n    else: \n        return 1 / (1 +np.exp(-X))\n\ndef gdnn_prime(X, activation_function):\n    leak_factor = 1/10000\n    if activation_function == 'tanh':\n        return 1-np.power(X,2)\n    if activation_function == 'lReLU':\n        return ((X > 0) * 1) + ((X <= 0)* leak_factor)\n    else: \n        return (1 / (1 +np.exp(-X)))*(1-(1 / (1 +np.exp(-X))))\n\n# Cost \ndef get_dnn_cost(Y_hat, Y):\n    #print(Y.shape)\n    m = Y.shape[1]\n    logprobs = np.multiply(np.log(Y_hat),Y) + np.multiply(np.log(1-Y_hat),1-Y)\n    cost = - np.sum(logprobs) /m\n    return cost\n    \n# Forward Propagation \ndef forward_dnn_propagation(X, params):\n    # Get Network Parameters \n    n = params[\"n\"]\n    L = len(n)\n    \n    A_prev = X\n    cache = {}\n    cache[\"A\"+str(0)] = X\n    for l in range(1,L):\n        W = params[\"W\"+str(l)]\n        b = params[\"b\"+str(l)]\n        Z = np.dot(W,A_prev)+b\n        A = gdnn(Z,params['act'+str(l)])\n        cache[\"Z\"+str(l)] = Z\n        cache[\"A\"+str(l)] = A\n        \n        A_prev = A\n    return A, cache \n\n# Backward Propagation\ndef back_dnn_propagation(X, Y, params, cache, alpha = 0.01, _lambda=0, keep_prob=1):\n    n = params[\"n\"]\n    L = len(n) -1\n    \n    m = X.shape[1]\n    W_limit = 5\n    A = cache[\"A\"+str(L)]\n    A1 = cache[\"A\"+str(L-1)]\n    #print(\"back_dnn_propagation: A(L) shape\"+str(A.shape))\n    #print(\"back_dnn_propagation: A1(L) shape\"+str(A1.shape))\n    grads = {}\n    \n    # Outer Layer \n    dZ = A - Y#gdnn_prime(A - Y, params[\"act\"+str(L)])\n    #print(\"back_dnn_propagation: dZ(L) shape\"+str(dZ.shape))\n    dW = 1/m * np.dot(dZ, A1.T)\n    #print(\"back_dnn_propagation: dW(L) shape\"+str(dW.shape))\n    db = 1/m * np.sum(dZ, axis=1, keepdims=True)\n    grads[\"dZ\"+str(L)] = dZ\n    grads[\"dW\"+str(L)] = dW + _lambda/m * params[\"W\"+str(L)]\n    grads[\"db\"+str(L)] = db\n    \n    # Update Outer Layer\n    params[\"W\"+str(L)] -= alpha * dW\n    #params[\"W\"+str(L)] = np.clip(params[\"W\"+str(L)],-W_limit,W_limit)\n    params[\"b\"+str(L)] -= alpha * db\n    for l in reversed(range(1,L)):\n        #dZ2 = A2 - Y\n        #dW2 = 1/m * np.dot(dZ2, A1.T)\n        #db2 = 1/m * np.sum(dZ2, axis=1, keepdims=True)\n        #dZ1 = np.dot(W2.T, dZ2)*g_prime(A1)\n        #dW1 = 1/m * np.dot(dZ1, X.T)\n        #db1 = 1/m * np.sum(dZ1, axis=1, keepdims=True)\n        \n        dZ2 = dZ\n        W2 = params[\"W\"+str(l+1)]\n        b = params[\"b\"+str(l)]\n        A2 = cache[\"A\"+str(l)]\n        A1 = cache[\"A\"+str(l-1)]\n        d = np.random.randn(A1.shape[0],A1.shape[1]) > keep_prob\n        A1 = A1 * d/keep_prob\n        dZ = np.dot(W2.T, dZ2)*gdnn_prime(A2, params[\"act\"+str(l)])\n\n        dW = 1/m * np.dot(dZ, A1.T) + _lambda/m * params[\"W\"+str(l)]\n        \n        db = 1/m * np.sum(dZ, axis=1, keepdims=True)\n        grads[\"dZ\"+str(l)] = dZ\n        grads[\"dW\"+str(l)] = dW\n        grads[\"db\"+str(l)] = db\n        params[\"W\"+str(l)] -= alpha *dW\n        #params[\"W\"+str(l)] = np.clip(params[\"W\"+str(l)],-W_limit,W_limit)\n        params[\"b\"+str(l)] -= alpha *db\n    \n    return grads, params    \n\ndef batch_back_propagation(X, Y, params, cache, alpha = 0.01, _lambda=0, keep_prob=1,batch_size=128):\n    # slice input and output data into smaller chunks \n    m = X.shape[1]\n    include_probability = keep_prob\n    idx_from = 0\n    idx_to = min(batch_size, m)\n    X_train = X[:,idx_from:idx_to]\n    y_train = Y[:,idx_from:idx_to]\n    while idx_to < m:\n        #print(\"Epoch from {} to {}\".format(idx_from, idx_to))\n        if np.random.random() < include_probability:\n            A, cache = forward_dnn_propagation(X_train, params)\n            grads, params= back_dnn_propagation(X_train, y_train, params, cache, alpha ,_lambda, keep_prob)    \n        idx_from += batch_size\n        idx_from = min(m, idx_from)\n        idx_to += batch_size\n        idx_to = min(m, idx_to)\n        if idx_from < idx_to:\n            X_train = X[:,idx_from:idx_to]\n            y_train = Y[:,idx_from:idx_to]\n    return grads, params","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0732d804-1d65-4653-b183-9abb08af5fc9","_uuid":"8264077a83d94dfdb63226a35248c14a5a437676","trusted":false,"collapsed":true},"cell_type":"code","source":"X2 = get_features(train_raw)\nplt.imshow(np.reshape(X2,(int(np.sqrt(X2.shape[1])),int(np.sqrt(X2.shape[1])))))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}