{"cells": [{"metadata": {"_cell_guid": "7e5b6bd4-ab43-45dc-825e-03e05a2e10d1", "_uuid": "86c3f1d7587cc2bb0d942cb9557444e88da2b6bc"}, "execution_count": null, "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "from sklearn.model_selection import KFold\n", "import matplotlib.pyplot as plt # Visualization\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "cell_type": "code", "outputs": []}, {"metadata": {"_cell_guid": "4296aaca-189e-44de-8682-95366007a650", "_uuid": "3cc51c0af847a0c2582a5a23f220bd0824c263c7"}, "execution_count": null, "source": ["# Load Data\n", "train_raw = pd.read_csv(\"../input/train.csv\")\n", "test_raw = pd.read_csv(\"../input/test.csv\")\n", "print(\"Data Loaded\")"], "cell_type": "code", "outputs": []}, {"metadata": {"collapsed": true, "_cell_guid": "1058620b-9a97-45f8-9d03-8a9c28914414", "_uuid": "ea8572f2f305d9afe287f791557938d7f6207ffd"}, "execution_count": null, "source": ["def get_features(raw_data):\n", "    cols = []\n", "    # Get data of each row from pixel0 to pixel783 \n", "    for px in range(784):\n", "        cols.append(\"pixel\"+str(px))   \n", "    return raw_data.as_matrix(cols)/255\n", "\n", "def cross_validated(X, n_samples):\n", "    kf = KFold(n_samples, shuffle = True)\n", "    result = [group for group in kf.split(X)]\n", "    return result        "], "cell_type": "code", "outputs": []}, {"metadata": {"collapsed": true, "_cell_guid": "633909b5-75bf-47af-acfb-6f13b5e1098a", "_uuid": "e356a012e7cca32c2ea030be631158dc9fe9405b"}, "execution_count": null, "source": ["# Deep Implementation \n", "# Initialize Parameters \n", "def init_dnn_parameters(n, activations, epsilons, filter=None):\n", "    L = len(n)\n", "    params = {}\n", "    for l in range(1,L):\n", "        W = np.random.randn(n[l],n[l-1]) * epsilons[l] \n", "        # Experiment, multiply filter in case of input layer weights \n", "        if filter1 is not None and l == 1:\n", "            W = np.dot(W, filter) \n", "        b = np.zeros((n[l],1))\n", "        params[\"W\"+str(l)] = W\n", "        #print(\" initizlizing W\"+str(l))\n", "        params[\"b\"+str(l)] = b                        \n", "        params[\"act\"+str(l)] = activations[l]\n", "    params[\"n\"] = n\n", "    return params\n", "\n", "# Activation Functions \n", "def gdnn(X, activation_function):\n", "    leak_factor = 1/10000\n", "    if activation_function == 'tanh':\n", "        return np.tanh(X)\n", "    if activation_function == 'lReLU':\n", "        return ((X > 0) * X) + ((X <= 0)* X * leak_factor)\n", "    else: \n", "        return 1 / (1 +np.exp(-X))\n", "\n", "def gdnn_prime(X, activation_function):\n", "    leak_factor = 1/10000\n", "    if activation_function == 'tanh':\n", "        return 1-np.power(X,2)\n", "    if activation_function == 'lReLU':\n", "        return ((X > 0) * 1) + ((X <= 0)* leak_factor)\n", "    else: \n", "        return (1 / (1 +np.exp(-X)))*(1-(1 / (1 +np.exp(-X))))\n", "\n", "# Cost \n", "def get_dnn_cost(Y_hat, Y):\n", "    #print(Y.shape)\n", "    m = Y.shape[1]\n", "    logprobs = np.multiply(np.log(Y_hat),Y) + np.multiply(np.log(1-Y_hat),1-Y)\n", "    cost = - np.sum(logprobs) /m\n", "    return cost\n", "    \n", "# Forward Propagation \n", "def forward_dnn_propagation(X, params):\n", "    # Get Network Parameters \n", "    n = params[\"n\"]\n", "    L = len(n)\n", "    \n", "    A_prev = X\n", "    cache = {}\n", "    cache[\"A\"+str(0)] = X\n", "    for l in range(1,L):\n", "        W = params[\"W\"+str(l)]\n", "        b = params[\"b\"+str(l)]\n", "        Z = np.dot(W,A_prev)+b\n", "        A = gdnn(Z,params['act'+str(l)])\n", "        cache[\"Z\"+str(l)] = Z\n", "        cache[\"A\"+str(l)] = A\n", "        \n", "        A_prev = A\n", "    return A, cache \n", "\n", "# Backward Propagation\n", "def back_dnn_propagation(X, Y, params, cache, alpha = 0.01, _lambda=0, keep_prob=1):\n", "    n = params[\"n\"]\n", "    L = len(n) -1\n", "    \n", "    m = X.shape[1]\n", "    W_limit = 5\n", "    A = cache[\"A\"+str(L)]\n", "    A1 = cache[\"A\"+str(L-1)]\n", "    #print(\"back_dnn_propagation: A(L) shape\"+str(A.shape))\n", "    #print(\"back_dnn_propagation: A1(L) shape\"+str(A1.shape))\n", "    grads = {}\n", "    \n", "    # Outer Layer \n", "    dZ = A - Y#gdnn_prime(A - Y, params[\"act\"+str(L)])\n", "    #print(\"back_dnn_propagation: dZ(L) shape\"+str(dZ.shape))\n", "    dW = 1/m * np.dot(dZ, A1.T)\n", "    #print(\"back_dnn_propagation: dW(L) shape\"+str(dW.shape))\n", "    db = 1/m * np.sum(dZ, axis=1, keepdims=True)\n", "    grads[\"dZ\"+str(L)] = dZ\n", "    grads[\"dW\"+str(L)] = dW + _lambda/m * params[\"W\"+str(L)]\n", "    grads[\"db\"+str(L)] = db\n", "    \n", "    # Update Outer Layer\n", "    params[\"W\"+str(L)] -= alpha * dW\n", "    #params[\"W\"+str(L)] = np.clip(params[\"W\"+str(L)],-W_limit,W_limit)\n", "    params[\"b\"+str(L)] -= alpha * db\n", "    for l in reversed(range(1,L)):\n", "        #dZ2 = A2 - Y\n", "        #dW2 = 1/m * np.dot(dZ2, A1.T)\n", "        #db2 = 1/m * np.sum(dZ2, axis=1, keepdims=True)\n", "        #dZ1 = np.dot(W2.T, dZ2)*g_prime(A1)\n", "        #dW1 = 1/m * np.dot(dZ1, X.T)\n", "        #db1 = 1/m * np.sum(dZ1, axis=1, keepdims=True)\n", "        \n", "        dZ2 = dZ\n", "        W2 = params[\"W\"+str(l+1)]\n", "        b = params[\"b\"+str(l)]\n", "        A2 = cache[\"A\"+str(l)]\n", "        A1 = cache[\"A\"+str(l-1)]\n", "        d = np.random.randn(A1.shape[0],A1.shape[1]) > keep_prob\n", "        A1 = A1 * d/keep_prob\n", "        dZ = np.dot(W2.T, dZ2)*gdnn_prime(A2, params[\"act\"+str(l)])\n", "\n", "        dW = 1/m * np.dot(dZ, A1.T) + _lambda/m * params[\"W\"+str(l)]\n", "        \n", "        db = 1/m * np.sum(dZ, axis=1, keepdims=True)\n", "        grads[\"dZ\"+str(l)] = dZ\n", "        grads[\"dW\"+str(l)] = dW\n", "        grads[\"db\"+str(l)] = db\n", "        params[\"W\"+str(l)] -= alpha *dW\n", "        #params[\"W\"+str(l)] = np.clip(params[\"W\"+str(l)],-W_limit,W_limit)\n", "        params[\"b\"+str(l)] -= alpha *db\n", "    \n", "    return grads, params    \n", "\n", "def batch_back_propagation(X, Y, params, cache, alpha = 0.01, _lambda=0, keep_prob=1,batch_size=128):\n", "    # slice input and output data into smaller chunks \n", "    m = X.shape[1]\n", "    include_probability = keep_prob\n", "    idx_from = 0\n", "    idx_to = min(batch_size, m)\n", "    X_train = X[:,idx_from:idx_to]\n", "    y_train = Y[:,idx_from:idx_to]\n", "    while idx_to < m:\n", "        #print(\"Epoch from {} to {}\".format(idx_from, idx_to))\n", "        if np.random.random() < include_probability:\n", "            A, cache = forward_dnn_propagation(X_train, params)\n", "            grads, params= back_dnn_propagation(X_train, y_train, params, cache, alpha ,_lambda, keep_prob)    \n", "        idx_from += batch_size\n", "        idx_from = min(m, idx_from)\n", "        idx_to += batch_size\n", "        idx_to = min(m, idx_to)\n", "        if idx_from < idx_to:\n", "            X_train = X[:,idx_from:idx_to]\n", "            y_train = Y[:,idx_from:idx_to]\n", "    return grads, params"], "cell_type": "code", "outputs": []}, {"metadata": {"_cell_guid": "0732d804-1d65-4653-b183-9abb08af5fc9", "_uuid": "8264077a83d94dfdb63226a35248c14a5a437676"}, "execution_count": null, "source": ["X2 = get_features(train_raw)\n", "\n", "labels = np.array(train_raw['label'])\n", "#labels = np.reshape(labels,(-1,1))\n", "m = labels.shape[0]\n", "y = np.zeros((m,10))\n", "for j in range(10):\n", "    y[:,j]=(labels==j)*1\n", "\n", "k = 38\n", "folds = 3\n", "oinst = 1\n", "h_layers = 5\n", "np.random.seed(1)\n", "cv_groups = cross_validated(X2, folds)\n", "\n", "alphas = np.linspace(0.045, 0.045, oinst)\n", "epsilons = np.linspace(0.76,0.78,oinst)\n", "gammas =  np.linspace(0.01,0.01,oinst)\n", "lambdas=  np.linspace(25.91,25.91,oinst)\n", "keep_probs=  np.linspace(0.9,0.9,oinst)\n", "alph_decays = np.linspace(0.995,0.995,oinst) \n", "n_1 = []\n", "break_tol = 0.00000001\n", "etscost = []\n", "etrcost= []\n", "seeds = []\n", "layers = []\n", "for j in range(oinst):\n", "    batch_processing = True\n", "    batch_size = 1024\n", "\n", "    #X = kernel(X2,X2,y,k,epsilon) # RBF\n", "    X = X2 # Direct Map\n", "    n = [X.shape[1]]\n", "    acts = ['input']\n", "    gamma = [0]\n", "    for layer in range(h_layers):\n", "        #n.append((10+h_layers-layer)**2) #((28-layer*3))**2)\n", "        n.append((14-layer)**2) #((28-layer*3))**2)\n", "        acts.append('lReLU') #tanh')\n", "        #gamma.append(gammas[j]/(2**(h_layers-layer)))\n", "        gamma.append(np.sqrt(2/n[layer-1]))\n", "    layers.append(j+1)    \n", "    n.append(y.shape[1])\n", "    acts.append('sigmoid')\n", "    #gamma.append(gammas[j])\n", "    gamma.append(np.sqrt(1/n[layer-1]))\n", "    #wall = 23\n", "    n_1.append(j+4)\n", "    np.random.seed(1)\n", "    iterations = 150\n", "    \n", "    alpha = alphas[j]#0.166# \n", "    \n", "    _lambda = lambdas[j] # 0.5#\n", "    keep_prob = keep_probs[j]\n", "    epsilon = 0.76#epsilons[j] #0.02 \n", "    \n", "    #n = [X.shape[1],wall,wall-1,wall-2,n_1[-1] ,16,1]\n", "    L = len(n) - 1\n", "    #acts = ['x','lReLU', 'lReLU',  'lReLU', 'lReLU', 'lReLU','sigmoid']\n", "\n", "    # Prepare Training and testing sets \n", "    X_train = X[cv_groups[0][0],:].T \n", "    y_train = y[cv_groups[0][0],:].T \n", "    labels_train = labels[cv_groups[0][0]]\n", "    # Experiment - Filter based on linear correlation \n", "    depth = 1024\n", "    filter1 = np.zeros((n[0],n[0]))\n", "    for dim in range(10):\n", "        for monomial in range(1,min(3, h_layers)):\n", "            X_sample = X_train[:,:depth].T**monomial\n", "            X_mean = np.reshape(np.mean(X_sample,axis=0),(1,-1))\n", "            y_sample = np.reshape(y_train[dim, :depth],(-1,1))\n", "\n", "            y_mean = np.mean(y_sample)\n", "            y_var = (y_sample - y_mean)*X_sample**0\n", "            numer = (np.dot((X_sample-X_mean).T,y_var))\n", "            denom = np.sqrt(np.sum(np.dot((X_sample-X_mean).T,(X_sample-X_mean))))*np.sqrt(np.dot((y_sample - y_mean).T,(y_sample - y_mean)))\n", "            filter1 += np.diag((numer/denom)[:,0])\n", "    filter1 /= np.linalg.norm(filter1)\n", "    filter2 = 1*(np.abs(filter1) > 0.0001 )\n", "    params = init_dnn_parameters(n, acts,gamma, filter2)\n", "\n", "    # Experiment \n", "    \n", "    X_test = X[cv_groups[0][1],:].T \n", "    y_test = y[cv_groups[0][1],:].T\n", "    print(\"Exp[{}] - Eps = {}, Alph = {}, Decay = {}, lambda={}\".format(j, epsilon, alpha,alph_decays[j], _lambda))\n", "    print(\"k = {}, |X| = {}, max(i) = {}\".format( k, X_test.shape[0], iterations))\n", "    print(\"Keep Prob = {}, gamma = {}\".format(keep_prob, gamma))\n", "    print(\"Network {} {}\".format(n,acts))\n", "    cost = []\n", "    tcost=[]\n", "    A, cache = forward_dnn_propagation(X_train, params)\n", "    for i in range(iterations):\n", "        if batch_processing:\n", "            grads, params = batch_back_propagation(X_train, \n", "                                                   y_train, \n", "                                                   params, \n", "                                                   cache, \n", "                                                   alpha,\n", "                                                   _lambda, \n", "                                                   keep_prob,                                                  \n", "                                                   batch_size)\n", "            A, cache = forward_dnn_propagation(X_train, params)\n", "            cost.append(np.mean(get_dnn_cost(A, y_train)))\n", "            A2, vectors = forward_dnn_propagation(X_test, params)\n", "            tcost.append(get_dnn_cost(A2, y_test))\n", "        else:\n", "            A, cache = forward_dnn_propagation(X_train, params)\n", "            cost.append(get_dnn_cost(A, y_train))\n", "            grads, params= back_dnn_propagation(X_train, y_train, params, cache, alpha ,_lambda, keep_prob)\n", "            A2, vectors = forward_dnn_propagation(X_test, params)\n", "            tcost.append(get_dnn_cost(A2, y_test))\n", "        \n", "        if alpha*np.abs(np.linalg.norm(grads[\"dW\"+str(L)])) < break_tol:\n", "            break\n", "        if i % 10 == 0:\n", "            alpha *= alph_decays[j]\n", "            print(\"---------------------------------------------------------------\")\n", "            print(\"i = {}, trc = {}, tsc={}, alph.dWL = {}\".format(i,cost[-1],\n", "                                                                   tcost[-1], \n", "                                                                   alpha*np.abs(np.linalg.norm(grads[\"dW\"+str(L)]))))\n", "            print(\" active alph = {}\".format(alpha))\n", "            print(\"Number Matching\")\n", "            for num in range(10):\n", "                y_hat = A2[num,:] > 0.5\n", "                y_star = y_test[num,:]\n", "                matched = np.sum((1-np.abs(y_star-y_hat))*y_star)\n", "                distance = np.linalg.norm((y_star - A2[num,:])*y_star)\n", "                m_test = sum(y_test[num,:]==1)\n", "                y_size = y_test.shape[1]\n", "                pct = matched/m_test\n", "                print(\"[{}] Matched {} {}% m_pos={}, Distance {}\".format(num, matched,pct*100, m_test,distance ))\n", "    etscost.append(tcost[-1])\n", "    etrcost.append(cost[-1])\n", "    plt.plot(tcost, label=\"testing case\"+str(j))\n", "    plt.plot(cost, '--',label=\"training case\"+str(j))\n", "    plt.legend()"], "cell_type": "code", "outputs": []}, {"metadata": {"_cell_guid": "a52dd4f5-0b50-45df-81a8-d57ebef69a69", "_uuid": "bf213554567e00397317057ff2ce0c1abdef69c3"}, "execution_count": null, "source": ["\n", "#print(X.shape)\n", "#print(params[\"W1\"].shape)\n", "#print(params[\"act1\"])\n", "L = len(n) \n", "#print(params[\"act\"+str(L-1)])\n", "if 1 ==1:\n", "    \n", "    plt.imshow(np.reshape(np.diag(filter2),(28,28)))\n", "    plt.colorbar()\n", "    plt.show()\n", "\n", "    for l in range(10):\n", "        im = X[l,:]\n", "        side = int(np.sqrt(im.shape[0]))\n", "        im = np.reshape(im,(side, side))\n", "        plt.figure()\n", "        plt.imshow(im, cmap='hot')\n", "        plt.show()\n", "    \n", "#for l in range(1,L):\n", "#if 1==1:\n", "    for l in range(1,min(10,L)):\n", "    \n", "        plt.imshow(params[\"W\"+str(l)])\n", "        plt.colorbar()\n", "        for node in range(min(25,params[\"W\"+str(l)].shape[0]-1)):\n", "            print(node, params[\"W\"+str(l)].shape[1])\n", "            W_node = params[\"W\"+str(l)][node]\n", "            side = int(np.sqrt(W_node.shape[0]))\n", "            W_node = np.reshape(W_node, (side,side))        \n", "            plt.figure()\n", "            im = plt.imshow(W_node, cmap='hot')   \n", "            #CS = plt.contour(W_node, cmap='gray')\n", "            #plt.clabel(CS, inline=5, fontsize=10)\n", "            plt.colorbar(im) \n", "            plt.title(\"Weight Vector of Node {} in Layer {}\".format(node, l))\n", "\n", "            plt.show()"], "cell_type": "code", "outputs": []}, {"metadata": {"_cell_guid": "5203cb45-a77c-40f6-9db4-cc10a94af143", "_uuid": "111e2202830de676501ef001a2ae87914b73cb06"}, "execution_count": null, "source": ["if 1==0:\n", "    print(y_test.shape)\n", "    print(A2[:,0:10]>0.5)\n", "    print(X2.shape)\n", "    plt.scatter(A2[:,0],y_test[:,0])"], "cell_type": "code", "outputs": []}, {"metadata": {"collapsed": true, "_cell_guid": "862426c4-9429-4abe-84d0-37bff4aa20af", "_uuid": "cd518aa11f3ec23dbc93c8a80cee55737281f05d"}, "execution_count": null, "source": ["X_test = get_features(test_raw)\n", "#test_raw.describe(include=\"all\")"], "cell_type": "code", "outputs": []}, {"metadata": {"_cell_guid": "e19d3315-9700-4110-9bae-60e25aacf48d", "_uuid": "c83c769ce4a322d0dd628851597c0b7b8665fcad"}, "execution_count": null, "source": ["A2, vectors = forward_dnn_propagation(X_test.T, params)\n", "# use A2 to construct \n", "\n", "y_hat = A2 > 0.5\n", "y_hat = y_hat*1\n", "\n", "data = np.zeros((y_hat.shape[1],1))\n", "for idx in range(y_hat.shape[0]):\n", "    data += idx * np.reshape(y_hat[idx,:],(-1,1))\n", "    \n", "#print(data)\n", "s1 = pd.Series(data[:,0])\n", "\n", "df = pd.DataFrame(data = s1)\n", "\n", "df.columns = ['Label']\n", "df.to_csv('deep_nn.csv', sep=',')"], "cell_type": "code", "outputs": []}, {"metadata": {"_cell_guid": "1f48bd2f-b947-41fe-9d3e-115dc0771150", "_uuid": "4f571161f43677ce95a91d4f42389deae43ceed7"}, "execution_count": null, "source": ["#print(X[0:1024,:].T.shape)\n", "#print(np.reshape(labels[:1024],(1,-1))*X[0:1024,:].T**0)\n", "#print(np.corrcoef(X[0:1024,0].T, np.reshape(labels[0:1024],(1,-1))))\n"], "cell_type": "code", "outputs": []}, {"metadata": {"collapsed": true}, "execution_count": null, "source": [], "cell_type": "code", "outputs": []}], "metadata": {"language_info": {"file_extension": ".py", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "codemirror_mode": {"version": 3, "name": "ipython"}, "version": "3.6.4", "nbconvert_exporter": "python", "name": "python"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}}, "nbformat": 4, "nbformat_minor": 1}