{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":16017,"databundleVersionId":708189,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import scipy.io\n# import mat73\nimport h5py\nimport matplotlib.pylab as plt\nimport pandas as pd\nimport numpy as np\nimport os\nimport cv2\n# from PIL import Image\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\nimport time\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"markdown","source":"Для виконання завдання було обрано набір даних Kannada-MNIST","metadata":{}},{"cell_type":"code","source":"!kaggle competitions download -c Kannada-MNIST -p C:\\Users\\eliza\\.kaggle\\","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!unzip C:\\Users\\eliza\\.kaggle\\Kannada-MNIST.zip -d C:\\Users\\eliza\\.kaggle\\KannadaMnist","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls C:\\Users\\eliza\\.kaggle\\KannadaMnist","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(r\"C:\\Users\\eliza\\.kaggle\\KannadaMnist\\train.csv\")\ntrain_df.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Image Preprocessing","metadata":{}},{"cell_type":"code","source":"labels = train_df['label'].values\nimages = train_df.drop(columns=['label'], axis=1).values\n\nprint(\"Images Shape: \", images.shape)\nprint(\"Label Shape: \", labels.shape)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Info about the dataset:**\n\nEach image is 28 pixels in height and 28 pixels in width, for a total of 784 pixels in total. Each pixel has a single pixel-value associated with it, indicating the lightness or darkness of that pixel, with higher numbers meaning darker. This pixel-value is an integer between 0 and 255, inclusive.","metadata":{}},{"cell_type":"code","source":"# if images contains file paths, we need to read each image using cv2.imread()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# preprocessed_images = []\n\n# for img in images:\n#     # Making each image 28 x 28 flattened vector\n#     img = img.reshape(28, 28)\n    \n#     # Resizing\n#     img = cv2.resize(img, (28, 28), interpolation=cv2.INTER_AREA)\n\n#     # Converting images to gray\n#     # img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) \n    \n#     # Normalizing to* range [0, 1]\n#     # img = img / 255\n\n#     # Reshaping Convert to Tensor Format (H, W, C) to (Batch, H, W, Channels (rgb))\n#     img = img.reshape(28, 28, 1)\n\n#     preprocessed_images.append(img)\n\n# X = np.array(preprocessed_images)\n# y = np.array(labels)\n\n# X = X / 255","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> Normalizing to range [0, 1]","metadata":{}},{"cell_type":"code","source":"X = np.array(images) / 255 \ny = np.array(labels)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# OneHotEncoder expects a 2D array\n# encoder = OneHotEncoder(sparse_output=False)\n# y_onehot = encoder.fit_transform(y.reshape(-1,1))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> One Hot Encoding Labels","metadata":{}},{"cell_type":"code","source":"# One Hot Encoding Labels\ndef onehot(y):\n    return np.eye(10)[y]\n\ny = onehot(y)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Images Shape: \", X.shape)\nprint(\"Label Shape: \", y.shape)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, ncols=3, figsize=(6, 6))\nax = ax.flatten()\n\nfor i in range(6):\n    ax[i].imshow(X[i].reshape(28,28), cmap=\"gray\")\n    ax[i].set_title(f'True: {np.argmax(y[i])}')   #y is one-hot encoded, so take argmax\n    ax[i].axis(\"off\")\n    \nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Test split","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = X_train.astype(np.float32) \nX_test = X_test.astype(np.float32)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Train Data: {X_train.shape}, {y_train.shape}\")\nprint(f\"Test Data: {X_test.shape}, {y_test.shape}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> X = (samples, input_size)\n\n> y = (samples, output_size) \n\n\nsamples = number of images in the dataset.\n\ninput_size = total number of pixels in a flattened image, which is 28×28=784","metadata":{}},{"cell_type":"markdown","source":"# MLP Building","metadata":{}},{"cell_type":"code","source":"def tanh(x):\n    return np.tanh(x)\n\ndef tanh_derivative(x):\n    return 1 - np.tanh(x)**2\n\ndef relu(x):\n    return np.maximum(0, x)\n\ndef relu_derivative(x):\n    return np.where(x > 0, 1, 0)\n\ndef softmax(x):\n    e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))\n    return e_x / np.sum(e_x, axis=-1, keepdims=True)\n\ndef leaky_relu(x, alpha=0.01):\n    return np.where(x > 0, x, alpha * x)\n\ndef leaky_relu_derivative(x, alpha=0.01):\n    return np.where(x > 0, 1, alpha)\n\ndef elu(x, alpha=0.01):\n    return np.where(x > 0, x, alpha * (np.exp(x) - 1))\n\ndef elu_derivative(x, alpha=0.01):\n    return np.where(x > 0, 1, alpha * np.exp(x))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NeuralNetwork:\n    def __init__(self, input_size, hidden_size, output_size, learning_rate, activation):\n        self.input_size = input_size # img_size[0] * img_size[1]\n        self.hidden_size = hidden_size\n        self.output_size = output_size\n        self.learning_rate = learning_rate\n        \n        self.activation_function = self._get_activation_function(activation)\n        self.activation_derivative = self._get_activation_derivative(activation)\n\n        # Initialize weights\n        self.w1 = np.random.randn(self.input_size, self.hidden_size) * 0.01\n        self.w2 = np.random.randn(self.hidden_size, self.output_size) * 0.01\n\n        # Initialize the biases\n        self.b1 = np.zeros((1, self.hidden_size))\n        self.b2 = np.zeros((1, self.output_size))\n\n        self.train_losses = []\n        self.train_accuracies = []\n        self.test_losses = []\n        self.test_accuracies = []\n    \n    def _get_activation_function(self, activation):\n        if activation == 'tanh':\n            return tanh\n        elif activation == 'relu':\n            return relu\n        elif activation == 'leaky_relu':\n            return leaky_relu\n        elif activation == 'elu':\n            return elu\n        else:\n            return relu # by default\n\n    def _get_activation_derivative(self, activation):\n        if activation == 'tanh':\n            return tanh_derivative\n        elif activation == 'relu':\n            return relu_derivative\n        elif activation == 'leaky_relu':\n            return leaky_relu_derivative\n        elif activation == 'elu':\n            return elu_derivative\n        else:\n            return relu_derivative # by default\n\n    # forward propagation\n    # zj = (wij * xi) + bi - activation ; a = F(zj) - activation function\n    def forward(self, X):\n        # Input to hidden layers (x -> h) h = x * W1 + b1\n        self.z1 = np.dot(X, self.w1) + self.b1\n        self.a1 = self.activation_function(self.z1)\n\n        # Hidden to output layers (h -> o) o = h * W2 + b2\n        self.z2 = np.dot(self.a1, self.w2) + self.b2\n        self.a2 = softmax(self.z2)\n        return self.a2\n\n    # Backpropagation\n    # Δwij = δj × Oj × η \n    def backward(self, X, y):\n        m = y.shape[0]\n        # error = self.a2 - y\n        dz2 = self.a2 - y     \n        dw2 = np.dot(self.a1.T, dz2) / m\n        db2 = np.sum(dz2, axis=0, keepdims=True) / m\n\n        dz1 = np.dot(dz2, self.w2.T) * self.activation_derivative(self.a1)\n        dw1 = np.dot(X.T, dz1) / m\n        db1 = np.sum(dz1, axis=0, keepdims=True) / m\n\n        self.w2 -= dw2 * self.learning_rate\n        self.b2 -= db2 * self.learning_rate  \n        self.w1 -= dw1 * self.learning_rate\n        self.b1 -= db1 * self.learning_rate\n        \n    def cross_entropy_loss(self, y, output):\n        m = y.shape[0]\n        loss = -np.sum(y * np.log(output + 1e-9)) / m\n        return loss         \n\n    def accuracy(self, y, output):\n        m = y.shape[0]\n        pred_label = np.argmax(output, axis=1)\n        true_label = np.argmax(y, axis=1)\n        accuracy = np.sum(pred_label == true_label) / m \n        return accuracy\n\n    def train(self, X_train, y_train, X_test, y_test, epochs=500):\n        for epoch in range(epochs):\n            # train data \n            output_train = self.forward(X_train)\n            self.backward(X_train, y_train)\n\n            train_loss = self.cross_entropy_loss(y_train, output_train)\n            train_accuracy = self.accuracy(y_train, output_train)\n            self.train_losses.append(train_loss)\n            self.train_accuracies.append(train_accuracy)\n            \n            # test data\n            output_test = self.forward(X_test)\n\n            test_loss = self.cross_entropy_loss(y_test, output_test)\n            test_accuracy = self.accuracy(y_test, output_test)\n            self.test_losses.append(test_loss)\n            self.test_accuracies.append(test_accuracy)\n\n            if epoch % 499 == 0:\n                print(f\"\\n|For Epoch {epoch}|\")\n                print(f\"Train Loss: {train_loss:.4f}, Train Accuracy: {train_accuracy * 100:.2f}%\")\n                print(f\"Test Loss: {test_loss:.4f}, Test Accuracy: {test_accuracy * 100:.2f}%\")\n\n    def predict(self, X):\n        output = self.forward(X)                \n        return np.argmax(output, axis=1)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_performance(model):\n    fig, ax = plt.subplots(2,1, figsize=(6,6))\n    ax[0].plot(model.train_losses, label = \"Train Loss\")\n    ax[0].plot(model.test_losses, label = \"Test Loss\")\n    ax[0].legend()\n    ax[0].set_xlabel(\"Epoch\")\n    ax[0].set_ylabel(\"Loss\")\n    ax[0].grid()\n    \n    ax[1].plot(model.train_accuracies, label=\"Train Accuracy\")\n    ax[1].plot(model.test_accuracies, label=\"Test Accuracy\")\n    ax[1].legend()\n    ax[1].set_xlabel(\"Epoch\")\n    ax[1].set_ylabel(\"Accuracy\")\n    ax[1].grid()\n    \n    plt.tight_layout()\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_predictions(X_test, y_test, y_pred, num_samples=6):\n    inc_idx = [i for i in range(len(y_test)) if np.argmax(y_test[i]) != y_pred[i]]\n\n    if len(inc_idx) == 0:\n        print(\"No Incorrect Labels: \")\n        return\n    else:\n        print(\"Num of Incorrect Labels: \", len(inc_idx))\n    \n    fig, ax = plt.subplots(2, 3, figsize=(6, 8))  \n    ax = ax.flatten()\n\n    for i, idx in enumerate(inc_idx[:num_samples]):\n        image = X_test[idx].reshape(28, 28)  \n        true_label = np.argmax(y_test[idx])  # Якщо y_test у One-Hot Encoding\n        predicted_label = y_pred[idx]\n                \n        ax[i].imshow(image, cmap='gray')\n        ax[i].set_title(f\"True: {true_label} | Pred: {predicted_label}\")\n        ax[i].axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> Firstly, we will build few neural networks to see its behaviour according to the learning rate parameter and two activation functions ('tanh', 'relu')","metadata":{}},{"cell_type":"code","source":"activations = ['tanh', 'relu'] #, \"leaky_relu\", 'elu']\nlearning_rates = [0.001, 0.01, 0.1]\nstatistics = []\n\nfor activation in activations:\n    print(f\"\\n-- Activation: {activation} --\")\n    for learning_rate in learning_rates:\n        print(f\"-- Learning Rate: {learning_rate} --\")\n        \n        nn = NeuralNetwork(input_size=X_train.shape[1], \n                           hidden_size=64, \n                           output_size=y_train.shape[1], \n                           learning_rate=learning_rate, \n                           activation=activation)\n        \n        start = time.time()\n        nn.train(X_train, y_train, X_test, y_test)\n        time_train = time.time() - start\n\n        start = time.time()\n        y_pred = nn.predict(X_test)\n        time_pred = time.time() - start\n        \n        test_accuracy = nn.accuracy(y_test, onehot(y_pred)) * 100\n        print(f\"\\nPrediction Accuracy: {test_accuracy:.2f}%\")\n        \n        statistics.append([activation, learning_rate, time_train, time_pred, test_accuracy])\n\n        plot_performance(nn)\n        plot_predictions(X_test, y_test, y_pred)\n\n# statistics_df = pd.DataFrame(statistics, columns=[\"Activation\", \"Learning Rate\" \"Time Train\", \"Time Pred\", \"Final Accuracy\"]).set_index([\"Activation\",\"Learning Rate\"])\n# statistics_df","metadata":{"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"statistics_df = pd.DataFrame(statistics, columns=[\"Activation\", \"Learning Rate\", \"Time Train\", \"Time Pred\", \"Final Accuracy\"]).set_index([\"Activation\",\"Learning Rate\"])\nstatistics_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> For **learning_rate = 0.001** the network showed the worst results. The accuracy was about 42% for tanh and noticeably less for relu, about 19%.\n>\n> For **learning_rate = 0.01** the accuracy increased significanty. Depending to the activation functions, it rose to approximately 67% for tanh and soared at 51% for relu.\n>\n> For **learning_rate = 0.1** the network was about 95% for both activation functions, which is preaty appealing result.\n","metadata":{}},{"cell_type":"markdown","source":"> For the **tanh** training time varieded between 427 ms to 436 ms, and prediction time was approximatelly 0.09 ms. For the **relu** training time varieded between 339 ms to 343 ms, and prediction time was approximatelly 0.07 ms. For both activation functions training time decreased eith the rose of learning_rate.\n\n> The least number of inorrect predictions was 632 for tanh and 629 for relu.","metadata":{}},{"cell_type":"markdown","source":"> From the DataFrame it can be seen that network works better with the higher value for the learning_rate, starting from 0.1. Additionally, relu activation function works slightly better that tanh.\n> Let's build several MLP networks with the next hidden layers: 'relu', \"leaky_relu\", 'elus' and see which one has the highest accuracy. Also, I will increase the value for the learning_rate, starting from 0.7.","metadata":{}},{"cell_type":"code","source":"def model_building(activations, learning_rates):\n    for activation in activations:\n        print(f\"\\n-- Activation: {activation} --\")\n        for learning_rate in learning_rates:\n            print(f\"-- Learning Rate: {learning_rate} --\")\n            \n            nn = NeuralNetwork(input_size=X_train.shape[1], \n                               hidden_size=64, \n                               output_size=y_train.shape[1], \n                               learning_rate=learning_rate, \n                               activation=activation)\n            \n            start = time.time()\n            nn.train(X_train, y_train, X_test, y_test)\n            time_train = time.time() - start\n    \n            start = time.time()\n            y_pred = nn.predict(X_test)\n            time_pred = time.time() - start\n            \n            test_accuracy = nn.accuracy(y_test, onehot(y_pred)) * 100\n            print(f\"\\nPrediction Accuracy: {test_accuracy:.2f}%\")\n            \n            statistics.append([activation, learning_rate, time_train, time_pred, test_accuracy])\n    \n            plot_performance(nn)\n            plot_predictions(X_test, y_test, y_pred)\n\n\nstatistics = []\nactivations = ['relu', \"leaky_relu\", 'elu']\nlearning_rates = [0.7, 1.4]\n\nmodel_building(activations, learning_rates)\n\nstatistics_df = pd.DataFrame(statistics, columns=[\"Activation\", \"Learning Rate\", \"Time Train\", \"Time Pred\", \"Final Accuracy\"]).set_index([\"Activation\",\"Learning Rate\"])\nstatistics_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"statistics_df = pd.DataFrame(statistics, columns=[\"Activation\", \"Learning Rate\", \"Time Train\", \"Time Pred\", \"Final Accuracy\"]).set_index([\"Activation\",\"Learning Rate\"])\nstatistics_df","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(r\"C:\\Users\\eliza\\.kaggle\\KannadaMnist\\test.csv\")\ntest_df.head(5)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_data_predict = test_df.drop(columns='id', axis=1).values","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_data_predict.shape","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"activation = 'elu'\nlearning_rate = 1.4\n\nnn = NeuralNetwork(input_size=X_data_predict.shape[1], \n                   hidden_size=64, \n                   output_size=10, \n                   learning_rate=learning_rate, \n                   activation=activation)\n\nnn.train(X_train, y_train, X_test, y_test)\n\ny_data_predict = nn.predict(X_data_predict)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_data_predict.size","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(6,6))\nfor i in range(10):\n    idx = np.random.randint(0, X_data_predict.shape[0])\n    image = X_data_predict[idx].reshape(28, 28) \n    predicted_label = y_data_predict[idx]\n    \n    plt.subplot(5, 5, i + 1)\n    plt.imshow(image, cmap='gray')\n    plt.title(f\"Pred: {predicted_label}\")\n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['label'] = y_data_predict\ntest_df.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result_df = test_df[['id', 'label']]\nresult_df.head(10)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result_df.to_csv('kannada-mnist-results.csv', index=False)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle competitions submit -c Kannada-MNIST -f kannada-mnist-results.csv -m \"Kannada-MNIST submission\"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}