{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport time\nimport sys\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nseed = 782\nnp.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntrain = df.to_numpy()\n\ntrain_y = train[:,0].astype('int8')\ntrain_x = train[:,1:].astype('float64')\n\ntrain = None\n\nprint(\"Shape Train Images: (%d,%d)\" % train_x.shape)\nprint(\"Shape Labels: (%d)\" % train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/digit-recognizer/test.csv\")\ntest = df.to_numpy().astype('float64')\nprint(\"Shape Test Images: (%d,%d)\" % test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image(image, shape, label=\"\", cmp=None):\n    img = np.reshape(image,shape)\n    plt.imshow(img,cmap=cmp, interpolation='none')\n    plt.title(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(12,10))\n\ny, x = 5,10\nfor i in range(0,(y*x)):\n    plt.subplot(y, x, i+1)\n    ni = np.random.randint(0,train_x.shape[0],1)[0]\n    show_image(train_x[ni],(28,28), train_y[ni], cmp=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_exemple_per_digit(exemples):\n    hist = np.ones(10)\n\n    for y in exemples:\n        hist[y] += 1\n\n    colors = []\n    for i in range(10):\n        colors.append(plt.get_cmap('viridis')(np.random.uniform(0.0,1.0,1)[0]))\n\n    bar = plt.bar(np.arange(10), hist, 0.8, color=colors)\n\n    plt.grid()\n    plt.show()\n\ncount_exemple_per_digit(train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num = 1e9\n\nfor i in range(1000000):\n    num += 1e-6\n\nnum -= 1e9\n\nprint(num)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalization(x, mu, sigma):\n    \n    x_norm = np.zeros_like(x)\n\n    for n in range(len(x)):\n        for j in range(len(x[n])):\n            if(sigma[j]!=0):\n                x_norm[n,j] = (x[n,j] - mu[j]) / sigma[j]\n            else:\n                x_norm[n,j] = 0\n                    \n    return x_norm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mu = np.mean(train_x, axis=0)\nsigma = np.max(train_x, axis=0)-np.min(train_x, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = normalization(test, mu, sigma)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x = normalization(train_x,mu, sigma)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Test Min: %.2f\" % np.min(test))\nprint(\"Test Max: %.2f\" % np.max(test))\nprint(\"Train Min: %.2f\" % np.min(train_x))\nprint(\"Train Max: %.2f\" % np.max(train_x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(12,10))\n\ny, x = 5,10\nfor i in range(0,(y*x)):\n    plt.subplot(y, x, i+1)\n    ni = np.random.randint(0,train_x.shape[0],1)[0]\n    show_image(train_x[ni],(28,28), train_y[ni], cmp=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(12,10))\n\ny, x = 5,10\nfor i in range(0,(y*x)):\n    plt.subplot(y, x, i+1)\n    ni = np.random.randint(0,test.shape[0],1)[0]\n    show_image(test[ni],(28,28),  cmp=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y = pd.get_dummies(train_y).to_numpy()\nprint(train_y[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ReLu(x, derivative=False):\n    if(derivative==False):\n        return x*(x > 0)\n    else:\n        return 1*(x > 0)\n\nx = np.arange(20)-10\nrelu = ReLu(x)\n\nplt.plot(x, relu)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Softmax(x):\n    x -= np.max(x)\n    sm = (np.exp(x).T / np.sum(np.exp(x),axis=1)).T\n    return sm\n\nx = np.arange(20)-10\nsoftmax = Softmax([x])\n\nplt.plot(x, softmax[0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def CreateWeights():\n    ##Initialization of the Weights and the Biases with the random gaussian function with mean zeron, and variance between 1/sqtr(num_inputs_layer)\n    \n    ninputs = 784\n    wl1 = 500 ##Number of neurons in the first layer\n    wl2 = 300 ##Number of neurons in the second layer\n    nclass = 10 ##Numer of the class, in this case it is the number of the digits.\n    \n    #layer1\n    w1 = np.random.normal(0, ninputs**-0.5, [ninputs,wl1])\n    b1 = np.random.normal(0, ninputs**-0.5, [1,wl1])\n    \n    #Layer2\n    w2 = np.random.normal(0, wl1**-0.5, [wl1,wl2])\n    b2 = np.random.normal(0, wl1**-0.5, [1,wl2])\n\n    #Layer3\n    w3 = np.random.normal(0, wl2**-0.5, [wl2,nclass])\n    b3 = np.random.normal(0, wl2**-0.5, [1,nclass])\n    \n    return [w1,w2,w3,b1,b2,b3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Dropout(x, dropout_percent):\n    mask = np.random.binomial([np.ones_like(x)],(1-dropout_percent))[0]  / (1-dropout_percent)\n    return x*mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(weights, x, dropout_percent=0):\n    \n    w1,w2,w3,b1,b2,b3  = weights \n    \n    #1-Hidden Layer\n    first = ReLu(x@w1+b1)\n    first = Dropout(first, dropout_percent)\n\n    #2-Hidden Layer\n    second = ReLu(first@w2+b2)\n    second = Dropout(second, dropout_percent)\n    \n    #Output Layer\n    return [first, second, Softmax(second@w3+b3)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy(output, y):\n    hit = 0\n    output = np.argmax(output, axis=1)\n    y = np.argmax(y, axis=1)\n    for y in zip(output, y):\n        if(y[0]==y[1]):\n            hit += 1\n\n    p = (hit*100)/output.shape[0]\n    return p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(14, 4))\n\ny = np.arange(100)/100\n\ne = -np.log(y)\ne2 = -np.log(1-y)\n\nax0.plot(y, e)\nax0.set_title(\"For t (right label) = 1\")\nax0.set_xlabel(\"Predict\")\nax0.set_ylabel(\"Error\")\nax1.plot(y, e2)\nax1.set_title(\"For t (right label) = 0\")\nax1.set_xlabel(\"Predict\")\nax1.set_ylabel(\"Error\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def log2(x):\n    if(x!=0):\n        return np.log(x)\n    else:\n        return -np.inf\n    \ndef log(y):\n    return [[log2(nx) for nx in x]for x in y]\n\ndef cost(Y_predict, Y_right, weights, nabla):\n    w1,w2,w3,b1,b2,b3  = weights\n    weights_sum_square = np.mean(w1**2) + np.mean(w2**2) + np.mean(w3**2)\n    Loss = -np.mean(Y_right*log(Y_predict) + (1-Y_right)*log(1-Y_predict)) + nabla/2 *  weights_sum_square\n    return Loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"porcent_valid = 0.1\nVALID_SIZE = round(train_x.shape[0]*porcent_valid)\n\nindex_data = np.arange(train_x.shape[0])\nnp.random.shuffle(index_data)\n\nx_train = train_x[index_data[VALID_SIZE:]]\nx_valid = train_x[index_data[:VALID_SIZE]]\n\n\nd_train = train_y[index_data[VALID_SIZE:]]\nd_valid = train_y[index_data[:VALID_SIZE]]\n\ntrain_x = None\ntrain_y = None\n\nx_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def SGD(weights, x, t, outputs, eta, gamma, nabla, cache=None):\n    \n    w1,w2,w3,b1,b2,b3  = weights\n    \n    \n    if(cache==None):\n            vw1 = np.zeros_like(w1)\n            vw2 = np.zeros_like(w2)\n            vw3 = np.zeros_like(w3)\n            vb1 = np.zeros_like(b1)\n            vb2 = np.zeros_like(b2)\n            vb3 = np.zeros_like(b3)\n    else:\n        vw1,vw2,vw3,vb1,vb2,vb3 = cache\n    \n    first, second, y = outputs\n   \n    w3_delta = (t-y)\n   \n    w2_error = w3_delta@w3.T\n    \n    w2_delta = w2_error * ReLu(second,derivative=True)\n\n    w1_error = w2_delta@w2.T\n    w1_delta = w1_error * ReLu(first,derivative=True)\n    \n    eta = -eta/x.shape[0]\n \n    vw3 = gamma*vw3 + eta * (second.T@w3_delta + nabla*w3)\n    vb3 = gamma*vb3 + eta * w3_delta.sum(axis=0)\n\n    vw2 = gamma*vw2 + eta * (first.T@w2_delta + nabla*w2)\n    vb2 = gamma*vb2 + eta * w2_delta.sum(axis=0)\n\n    vw1 = gamma*vw1 + eta * (x.T@w1_delta + nabla*w1)\n    vb1 = gamma*vb1 + eta * w1_delta.sum(axis=0)\n    \n    \n    w3 -= vw3\n    b3 -= vb3\n\n    w2 -= vw2\n    b2 -= vb2\n\n    w1 -= vw1\n    b1 -= vb1\n    \n    weights = [w1,w2,w3,b1,b2,b3]\n    cache = [vw1,vw2,vw3,vb1,vb2,vb3]\n    \n    return weights, cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.ndimage.interpolation import map_coordinates\nfrom scipy.ndimage.filters import gaussian_filter\n\ndef elastic_transform(image, alpha, sigma, random_state=None):\n    \"\"\"Elastic deformation of images as described in [Simard2003]_.\n    .. [Simard2003] Simard, Steinkraus and Platt, \"Best Practices for\n       Convolutional Neural Networks applied to Visual Document Analysis\", in\n       Proc. of the International Conference on Document Analysis and\n       Recognition, 2003.\n    \"\"\"\n    if random_state is None:\n        random_state = np.random.RandomState(None)\n\n    shape = image.shape\n    dx = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode=\"constant\", cval=0) * alpha\n    dy = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode=\"constant\", cval=0) * alpha\n\n    x, y = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]))\n    indices = np.reshape(y+dy, (-1, 1)), np.reshape(x+dx, (-1, 1))\n\n    return map_coordinates(image, indices, order=1).reshape(shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_t = np.array([elastic_transform(xx.reshape(28,28),15,3).reshape(784) for xx in x_train[0:10]])\nplt.subplot(1,2,1)\nplt.imshow(x_t[0].reshape(28,28))\nplt.subplot(1,2,2)\nplt.imshow(x_train[0].reshape(28,28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(weights, x_train, y_train, x_valid, y_valid, epochs = 10, nbatchs=25, alpha = 1e-3, decay = 0, momentum = 0, l2 = 0.001, dropout_percent = 0):\n    \n    pross = x_train.shape[0]*0.05\n    \n    history = [[],[]]\n    \n    index = np.arange(x_train.shape[0])\n    cache = None\n    print(\"Train data: %d\" % (x_train.shape[0]))\n    print(\"Validation data: %d \\n\" % (x_valid.shape[0]))\n    mtime = 0\n    \n    r_weights = []\n    max_accuracy_valid = 0\n    \n    for j in range(epochs):\n        np.random.shuffle(index)\n        t = 0\n        iterations = round(x_train.shape[0]/nbatchs)\n        prog = \"\"\n        sacurr = 0\n        sloss = 0\n        sys.stdout.write(\"\\nEpochs: %2d \\ %2d \\n\"% (j+1,epochs))\n        stime = 0\n        timeIT = time.time()\n        for i in range(iterations):\n            timeI = time.time()\n            f = i*nbatchs\n            l = f+nbatchs\n            \n            if(l>(x_train.shape[0]-1)):\n                l = x_train.shape[0]\n                \n            x = np.array([elastic_transform(xx.reshape(28,28),15,3).reshape(784) for xx in x_train[index[f:l]]])\n            y = y_train[index[f:l]]\n\n            outputs = predict(weights, x, dropout_percent)\n            \n            loss = cost(outputs[-1], y, weights, l2)\n            \n            \n            accuracy_t = accuracy(outputs[-1], y)\n            \n            sacurr += accuracy_t\n            sloss += loss\n            \n            accuracy_train = sacurr/(i+1)\n            loss_train = sloss/(i+1)\n            \n            weights, cache = SGD(weights, x, y, outputs, alpha, momentum, l2, cache)\n            \n            t+= x.shape[0]\n            \n            qtd = round(t/pross)\n            prog = \"[\"\n            for p in range(20):\n                if(p<qtd-1):\n                    prog += \"=\"\n                elif(p==qtd-1):\n                    prog += \">\"\n                else:\n                    prog += \".\"\n            prog += \"]\"\n\n            \n            stime += time.time()-timeI\n            mtime = stime/(i+1)\n            mTimeT = mtime * (iterations-i-1)\n            \n            sys.stdout.write(\"\\r%5d/%5d %s ETA: %3d s - loss: %.4f  acc: %.4f\" % (t, x_train.shape[0], prog, mTimeT, loss_train, accuracy_train))\n            \n            history[0].append([loss_train, accuracy_train])\n        mtime = time.time()-timeIT\n        alpha = alpha - (alpha*decay)\n        \n        outputs = predict(weights, x_valid)\n        \n        loss_valid = cost(outputs[-1], y_valid, weights, l2)\n        accuracy_valid = accuracy(outputs[-1], y_valid)\n        \n        sys.stdout.write(\"\\r%5d/%5d %s ETA: %3d s loss: %.4f  acc: %.4f - lossValid: %.4f  accValid: %.4f \" % ( t, x_train.shape[0], prog, mtime, loss_train, accuracy_train, loss_valid, accuracy_valid))\n        history[1].append([loss_valid, accuracy_valid])\n            \n        if(accuracy_valid>=max_accuracy_valid):\n            w1,w2,w3,b1,b2,b3  = weights\n            r_weights = [w1.copy(),w2.copy(),w3.copy(),b1.copy(),b2.copy(),b3.copy()]\n            max_accuracy_valid = accuracy_valid\n        \n    return r_weights, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = CreateWeights()\n\nalpha = 5e-2\nepochs = 40\nnbatchs = 100\nweights, history = run(weights, \n              x_train, d_train, \n              x_valid, d_valid, \n              epochs = epochs,\n              nbatchs=nbatchs, \n              alpha = alpha, \n              decay = 0.05, \n              momentum = 0.9, \n              l2 = 1e-3, \n              dropout_percent = 0.20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history = np.array(history[0])\nt_loss = train_history[:,:1]\nt_acc = train_history[:,1:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_history = np.array(history[1])\ntrain_history.shape\nv_loss = valid_history[:,:1]\nv_acc = valid_history[:,1:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,10))\n\nplt.subplot(2, 1, 1)\nx = np.arange(epochs)*int(x_train.shape[0]/nbatchs)\nplt.plot(x,v_acc)\nplt.plot(t_acc)\n\nplt.subplot(2, 1, 2)\nplt.plot(x,v_loss)\nplt.plot(t_loss)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i in range(cm.shape[0]):\n        for j in range(cm.shape[1]):\n            plt.text(j, i, cm[i, j],\n                     horizontalalignment=\"center\",\n                     color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outputs = predict(weights, x_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = np.argmax(outputs[-1],axis=1)\npredict_dummies = pd.get_dummies(p).to_numpy().astype('int8')\n\ncm = np.zeros((10,10)).astype(np.int64)\n\nd_valid_int = np.argmax(d_valid, axis=1)\n\nfor i in range(predict_dummies.shape[0]):\n    cm[d_valid_int[i]] += predict_dummies[i]\n\nprint(cm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(cm, range(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s_cm = np.sum(cm,axis=0)\nfor i in range(10):\n    p = cm[i][i]/s_cm[i]\n    print(\"%d - %.3f %%\" % (i,p))\n\nprint(\" \");\nprint(\"%f %%\" % accuracy(outputs[-1], d_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w1,w2,w3,b1,b2,b3  = weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(12,10))\n\ny, x = 4,10\nfor i in range(0,(y*x)):\n    ni = np.random.randint(0,w1.shape[1],1)[0]\n    plt.subplot(y, x, i+1)\n    plt.imshow(w1[:,ni].reshape((28,28)))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(12,10))\n\ny, x = 4,10\nfor i in range(0,(y*x)):\n    ni = np.random.randint(0,w2.shape[1],1)[0]\n    plt.subplot(y, x, i+1)\n    plt.imshow(w2[:,ni].reshape((20,25)))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(12,10))\n\ny, x = 1,10\nfor i in range(0,(y*x)):\n    plt.subplot(y, x, i+1)\n    plt.imshow(w3[:,i].reshape((20,15)))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outputs = predict(weights, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = np.argmax(outputs[-1],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nplt.figure(figsize=(20,12))\n\ny, x = 5,11\nfor i in range(0,(y*x)):\n    plt.subplot(y, x, i+1)\n    ni = np.random.randint(0,test.shape[0],1)[0]\n    v = str(d[ni])\n    show_image(test[ni],(28,28), v, cmp=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ImageId,Label\n\npd.DataFrame({\"ImageId\": list(range(1,len(d)+1)), \"Label\": d}).to_csv('output.csv', index=False, header=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}