{"cells":[{"metadata":{"_uuid":"a9dc0ec4283962519f12dda774cd2203606154f1"},"cell_type":"markdown","source":"Dear Kagglers,\n\nIn this challenge i would like to share my knowledge that i gained from deeplearning.ai course. The implementation of digit classification will be performed with 3 differnt ways and libraries:\n1.  Logistic Regression with numpy (~=82% accuracy)\n2. Neural Network with tensorflow (~=89% accuracy)\n3. Convolution Neural Network with Keras (~=98% accuracy)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n#get dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0005b269284cdd3e734ed78f771500a00aaf5cd9"},"cell_type":"code","source":"import pandas as pd\ntrain_dataset = pd.read_csv('../input/train.csv')\ntest_dataset = pd.read_csv('../input/test.csv')\nprint(\"train dataset size: \", train_dataset.shape[0])\nprint(\"test dataset size: \", test_dataset.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c30ac9e3ab2fe138472da1618b6b89eeeb3266f9"},"cell_type":"markdown","source":"Split Dataset in Training and Cross Validation"},{"metadata":{"trusted":true,"_uuid":"6acfe25a8c03eca0fac896861db45301d936a09a"},"cell_type":"code","source":"\n#split train dataset\nimport numpy\nmsk = np.random.rand(len(train_dataset)) < 0.75\ntrain = train_dataset[msk]\ncv = train_dataset[~msk]\nprint(\"train dataset size: \", train.shape[0])\nprint(\"cross validation dataset size: \", cv.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f8d9e8282341ec3592a669d99fb7eaed2728b4e"},"cell_type":"markdown","source":"Normalize input and transform the label in binary format "},{"metadata":{"trusted":true,"_uuid":"ed071c38133ee7e0a4dc09a3ac3cf012c340f9fd"},"cell_type":"code","source":"#define labels and normalize inputs\nlabels_train = train[\"label\"]\nX_train = np.array(train.drop(\"label\",axis=1)) / 255\nlabels_cv = cv[\"label\"]\nX_cv = np.array(cv.drop(\"label\",axis=1)) / 255\n\n#convert labels to multi-class binaries\nfrom sklearn import preprocessing \nlb = preprocessing.LabelBinarizer()\ny_train = lb.fit_transform(labels_train)\ny_cv = lb.fit_transform(labels_cv)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fee21ddbfafa76019ecb00233af7214785a5326f"},"cell_type":"markdown","source":"Plot the distribution of dataset and define some constant variables"},{"metadata":{"trusted":true,"_uuid":"54493bb33e11880caa0d02b9223d233738d75aea"},"cell_type":"code","source":"\n#explole the range of numbers in dataset\nimport matplotlib.pyplot as plt\nfig, ax = plt.subplots(figsize=(6, 3), subplot_kw=dict(aspect=\"equal\"))\nax.pie(labels_train.value_counts(),labels = ['0','1','2','3','4','5','6','7','8','9'])\nfig, ax1 = plt.subplots(figsize=(6, 3), subplot_kw=dict(aspect=\"equal\"))\nax1.pie(labels_cv.value_counts(),labels = ['0','1','2','3','4','5','6','7','8','9'])\nplt.show()\n\n#define constant variables\nnum_of_train_examples = X_train.shape[0]\nnum_of_features = X_train.shape[1]\nnum_of_cv_examples = X_cv.shape[0]\nnum_of_classes = 10\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93daa2c78c762fbe537928f89b10fda1aa135a0d"},"cell_type":"markdown","source":"1.  Logistic Regression with numpy"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#initialize parameters\nw = np.zeros(num_of_features)\nb = 0\nparameters = {\n    \"weights\" : w,\n    \"bias\" : b\n}\n\n#initialize hyperparameters\nnum_iterations = 100\nreg_lambda = 0.005\nlearning_rate = 0.08\nhyperparameters = {\n    \"num_iterations\" : num_iterations,\n    \"lambda\" : reg_lambda,\n    \"alpha\" : learning_rate\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85f91c56d5a7b8e408b3498c9298eda044ff5538"},"cell_type":"code","source":"# Compute Hypothesis (Predict)\ndef forward_propagation(X,parameters):\n    w = parameters.get(\"weights\")\n    b = parameters.get(\"bias\")\n    z = np.dot(X,w) + b\n    h = 1 / (1 + np.exp(-z))\n    return h\n\n# compute Cost Function\ndef compute_log_reg_cost(h,y,m):\n    return -(np.dot(y.T,np.log(h)) + np.dot((1-y.T),np.log(1-h))) / m\n\n#compute gradients\ndef backward_propagation(X,y,h,hyperparameters,m):\n    learning_rate = hyperparameters.get(\"alpha\")\n    \n    error = h-y\n    grad_w = (learning_rate / m) * np.dot(X.T,error)\n    grad_b = (learning_rate / m) * sum(error)\n    \n    gradients = {\n        \"dw\" : grad_w,\n        \"db\" : grad_b\n    }\n    \n    return gradients\n\n#update parameters\ndef update_parameters(parameters,gradients):\n    dw = gradients.get(\"dw\")\n    db = gradients.get(\"db\")\n    w = parameters.get(\"weights\")\n    b = parameters.get(\"bias\")\n\n    w = w - dw\n    b = b - db\n    \n    parameters = {\n        \"weights\" : w,\n        \"bias\" : b\n    }\n    \n    return parameters\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e7a26173b9626bd76338424c02c8a69ebcaf656"},"cell_type":"code","source":"#model\ndef logistic_regression_model(X,y,m,n,parameters,hyperparameters):\n    num_iterations = hyperparameters.get(\"num_iterations\")\n    cost_list = []\n    for i in range(num_iterations):\n        hypothesis = forward_propagation(X,parameters)\n        cost = compute_log_reg_cost(hypothesis,y,m)\n        gradients = backward_propagation(X,y,hypothesis,hyperparameters,m)\n        parameters = update_parameters(parameters,gradients)\n        if i%20 == 0:\n            cost_list.append(cost)\n    \n    return (parameters,cost_list)\n\n#run model for each label\ndef multi_label_logistic_regression_model(X,Y,num_labels,m,n,parameters,hyperparameters):\n    W = []\n    B = []\n    for i in range(num_labels):\n        y = Y[: , i]\n        parameters,cost = logistic_regression_model(X,y,m,n,parameters,hyperparameters)\n        W.append(parameters.get(\"weights\"))\n        B.append(parameters.get(\"bias\"))\n    multi_parameters = {\n        \"W\" : np.array(W),\n        \"B\" : np.array(B)\n    }\n    return multi_parameters\n\nmulti_parameters = multi_label_logistic_regression_model(X_train,y_train,num_of_classes,num_of_train_examples,num_of_features,parameters,hyperparameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a71f641097b6af88cd4dd1c7f893ddaa11341719"},"cell_type":"code","source":"#predict - return the label with the bigger propability\ndef predict(X,multi_parameters):\n    W = multi_parameters.get(\"W\")\n    B = multi_parameters.get(\"B\")\n    z = np.dot(X,W.T) + B\n    predict = 1 / (1 + np.exp(-z))\n    max_idx = predict.argmax(axis=1)\n    return max_idx\ntrain_predictions = predict(X_train,multi_parameters)\ncv_predictions = predict(X_cv,multi_parameters)\n#evaluate results\ndef evaluation(predictions,Y):\n    total = Y.shape[0]\n    count_right_predict = 0\n    idx = 0\n    for predict in predictions:\n        if predict == Y[idx] : \n            count_right_predict = count_right_predict + 1\n        idx = idx + 1\n    return (count_right_predict/total)\ntrain_accuracy = evaluation(train_predictions,np.array(labels_train))\ncv_accuracy = evaluation(cv_predictions,np.array(labels_cv))\nprint('train_accuracy: ', train_accuracy)\nprint('cv_accuracy: ' , cv_accuracy)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4cfb6eaf1da4f55e2ede751f1ab6a7900367ac4b"},"cell_type":"markdown","source":"2. Neural Network with tensorflow\n"},{"metadata":{"trusted":true,"_uuid":"b0edb3915629531eb736806d667dd57a3d0eff71"},"cell_type":"code","source":"batch_size = 128\n\nn_hidden_1 = 256 # 1st layer number of neurons\nn_hidden_2 = 256 # 2nd layer number of neurons\nn_hidden_3 = 128 # 3nd layer number of neurons\n\nimport tensorflow as tf\n# Define the input function for training\ninput_fn = tf.estimator.inputs.numpy_input_fn(\n    x={'digits_images': np.array(X_train)}, y=np.array(labels_train),\n    batch_size=batch_size,  shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fef3c7274ee16c6f6fb35431b40f185e3e336c9"},"cell_type":"code","source":"# Define the neural network\ndef neural_net(x_dict):\n    # TF Estimator input is a dict, in case of multiple inputs\n    x = x_dict['digits_images']\n    # Hidden fully connected layer with 256 neurons\n    layer_1 = tf.layers.dense(x, n_hidden_1)\n    # Hidden fully connected layer with 256 neurons\n    layer_2 = tf.layers.dense(layer_1, n_hidden_2)\n    # Output fully connected layer with a neuron for each class\n    layer_3 = tf.layers.dense(layer_2, n_hidden_3)\n    # Output fully connected layer with a neuron for each class\n    out_layer = tf.layers.dense(layer_3, num_of_classes)\n    return out_layer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abb66686a7771a39cb47069ffa216d2fbfba85de"},"cell_type":"code","source":"# Define the model function (following TF Estimator Template)\ndef model_fn(features, labels, mode):\n    learning_rate = 0.1\n    \n    # Build the neural network\n    logits = neural_net(features)\n    \n    # Predictions\n    pred_classes = tf.argmax(logits, axis=1)\n    pred_probas = tf.nn.softmax(logits)\n    \n    # If prediction mode, early return\n    if mode == tf.estimator.ModeKeys.PREDICT:\n        return tf.estimator.EstimatorSpec(mode, predictions=pred_classes) \n        \n    # Define loss and optimizer\n    loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n        logits=logits, labels=tf.cast(labels, dtype=tf.int32)))\n    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n    train_op = optimizer.minimize(loss_op, global_step=tf.train.get_global_step())\n    \n    # Evaluate the accuracy of the model\n    acc_op = tf.metrics.accuracy(labels=labels, predictions=pred_classes)\n    \n    # TF Estimators requires to return a EstimatorSpec, that specify\n    # the different ops for training, evaluating, ...\n    estim_specs = tf.estimator.EstimatorSpec(\n      mode=mode,\n      predictions=pred_classes,\n      loss=loss_op,\n      train_op=train_op,\n      eval_metric_ops={'accuracy': acc_op})\n\n    return estim_specs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"635037ae2caff349fcff3d160dcd1cc59ca3ad45"},"cell_type":"code","source":"\n\n# Build the Estimator\nmodel = tf.estimator.Estimator(model_fn)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b41be5a94b2489e88898d432a34938ff70d4a1a2"},"cell_type":"code","source":"# Train the Model\nmodel.train(input_fn, steps=num_iterations)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72d592c68c980d1c0d8fa2b2fccb066afb762692"},"cell_type":"code","source":"#evaluate\nmodel.evaluate(input_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95076fa104209be479ca796b17ae009715235d79"},"cell_type":"code","source":"\n\n# Evaluate the Model\n# Define the input function for evaluating\ninput_fn = tf.estimator.inputs.numpy_input_fn(\n    x={'digits_images': np.array(X_cv)}, y=np.array(labels_cv),\n    batch_size=batch_size, shuffle=False)\n# Use the Estimator 'evaluate' method\nmodel.evaluate(input_fn)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1463722ceec8315c76c913b8c0e7751c0779c42"},"cell_type":"markdown","source":"3. Convolution Neural Network with Keras"},{"metadata":{"trusted":true,"_uuid":"a6c18000aaa51065d8eed93d87d904531d02bf48"},"cell_type":"code","source":"X_train = X_train.reshape(-1, 28,28, 1)\nX_cv = X_cv.reshape(-1, 28,28, 1)\nX_train.shape, X_cv.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbc2e9628340868c8505af1c16d1db62f7e6c60d"},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential,Input,Model\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.advanced_activations import LeakyReLU\n\nfashion_model = Sequential()\nfashion_model.add(Conv2D(32, kernel_size=(3, 3),activation='linear',input_shape=(28,28,1),padding='same'))\nfashion_model.add(LeakyReLU(alpha=0.1))\nfashion_model.add(MaxPooling2D((2, 2),padding='same'))\nfashion_model.add(Conv2D(64, (3, 3), activation='linear',padding='same'))\nfashion_model.add(LeakyReLU(alpha=0.1))\nfashion_model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))\nfashion_model.add(Conv2D(128, (3, 3), activation='linear',padding='same'))\nfashion_model.add(LeakyReLU(alpha=0.1))                  \nfashion_model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))\nfashion_model.add(Flatten())\nfashion_model.add(Dense(128, activation='linear'))\nfashion_model.add(LeakyReLU(alpha=0.1))                  \nfashion_model.add(Dense(num_of_classes, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"204022bc1a09f0a9f028f28b7a8432fc8d0c7844"},"cell_type":"code","source":"fashion_model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(),metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e470538b0976e2f2f4c18a70f49abe64d2f7c09"},"cell_type":"code","source":"fashion_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2376152683b5dacbaeff2a0ccaa316cfb80e59ad"},"cell_type":"code","source":"num_iterations = 5\nfashion_train = fashion_model.fit(X_train, y_train, batch_size=batch_size,epochs=num_iterations,verbose=1,validation_data=(X_cv, y_cv))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"246a4cd79995fcaae53eb9e7ac70f5aa7ff49965"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}