{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-16T14:31:31.517678Z","iopub.execute_input":"2024-08-16T14:31:31.518351Z","iopub.status.idle":"2024-08-16T14:31:51.122041Z","shell.execute_reply.started":"2024-08-16T14:31:31.518320Z","shell.execute_reply":"2024-08-16T14:31:51.120914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ideas\n\nhttps://www.kaggle.com/code/lainconn/digit-recog-with-kerastuner-python-keras3\n\nhttps://www.kaggle.com/code/krzysztofszafraski/simple-nn-and-visualizing-misclassified-examples","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt ","metadata":{"execution":{"iopub.status.busy":"2024-08-16T14:31:51.123603Z","iopub.execute_input":"2024-08-16T14:31:51.124137Z","iopub.status.idle":"2024-08-16T14:31:51.848836Z","shell.execute_reply.started":"2024-08-16T14:31:51.124109Z","shell.execute_reply":"2024-08-16T14:31:51.847859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-16T14:31:51.850082Z","iopub.execute_input":"2024-08-16T14:31:51.850380Z","iopub.status.idle":"2024-08-16T14:31:57.686202Z","shell.execute_reply.started":"2024-08-16T14:31:51.850354Z","shell.execute_reply":"2024-08-16T14:31:57.685411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train_df.columns.to_list()\ntarget = 'label'\nfeatures.remove(target)\n\nX = train_df[features]\ny = train_df[target]","metadata":{"execution":{"iopub.status.busy":"2024-08-16T14:31:57.688079Z","iopub.execute_input":"2024-08-16T14:31:57.688366Z","iopub.status.idle":"2024-08-16T14:31:57.787269Z","shell.execute_reply.started":"2024-08-16T14:31:57.688342Z","shell.execute_reply":"2024-08-16T14:31:57.786461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m, n = X.shape\n\nfig, axes = plt.subplots(5,5, figsize=(5,5))\nfig.tight_layout(pad=0.12,rect=[0, 0.03, 1, 0.9]) \n\nfor ax in axes.flat:\n    \n    # select random index from the data and reshape the given image to the original format 28x28\n    random_index = np.random.randint(m)\n    X_random_reshaped = X.iloc[random_index].values.reshape((28,28))\n    \n    # display the image\n    ax.imshow(X_random_reshaped, cmap='gray')\n    ax.set_title(y.iloc[random_index])\n    ax.set_axis_off()\n    fig.suptitle(\"Examples of handwrited digits from the training set\", fontsize=14)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T14:31:57.788348Z","iopub.execute_input":"2024-08-16T14:31:57.788633Z","iopub.status.idle":"2024-08-16T14:32:00.186657Z","shell.execute_reply.started":"2024-08-16T14:31:57.788608Z","shell.execute_reply":"2024-08-16T14:32:00.185776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalize input","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2137)\n\nscaler = StandardScaler()\nscaler.fit(X_train)\n\nX_train = scaler.transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T14:32:00.187971Z","iopub.execute_input":"2024-08-16T14:32:00.188397Z","iopub.status.idle":"2024-08-16T14:32:00.861852Z","shell.execute_reply.started":"2024-08-16T14:32:00.188362Z","shell.execute_reply":"2024-08-16T14:32:00.861009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_net(X_train, y_train, validation_data=None, nu1=50, nu2=25, nu3=10, activation='relu', regul1=0, regul2=0, \n              dropRate=0, optimizer='adam', epochs=10, batch_size=32):\n    \n    model = Sequential(\n    [   \n        tf.keras.Input(shape=(784,)),\n        Dense(units = nu1, activation = activation, kernel_regularizer = tf.keras.regularizers.L1L2(l1=regul1, l2=regul2)),\n        tf.keras.layers.Dropout(dropRate),\n        Dense(units = nu2, activation = activation, kernel_regularizer = tf.keras.regularizers.L1L2(l1=regul1, l2=regul2)),\n        tf.keras.layers.Dropout(dropRate),\n        Dense(units = nu3, activation = 'linear')   \n    ])\n    \n    # we used linear activation in the last layer, so now we have to set \"from_logits=True\" in the loss function\n    # it's considered to be more numerically stable\n    model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n                  optimizer=optimizer,\n                  metrics=['accuracy'])\n    \n    histModel = model.fit(x=X_train, y=y_train, batch_size=batch_size, epochs=epochs, validation_data=validation_data, verbose=0)\n    \n    TrainAccuracy = histModel.history['accuracy']\n    \n    if validation_data != None:\n        TestAccuracy = histModel.history['val_accuracy']\n        BestScore = max(TestAccuracy)\n    else: \n        TestAccuracy = []\n        BestScore = 0\n    \n    return model, histModel, TrainAccuracy, TestAccuracy, BestScore","metadata":{"execution":{"iopub.status.busy":"2024-08-16T14:32:00.862970Z","iopub.execute_input":"2024-08-16T14:32:00.863290Z","iopub.status.idle":"2024-08-16T14:32:00.872969Z","shell.execute_reply.started":"2024-08-16T14:32:00.863264Z","shell.execute_reply":"2024-08-16T14:32:00.871974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import layers\n\ndef train_net(X_train, y_train, validation_data=None, nu1=50, nu2=25, nu3=10, activation='relu', regul1=0, regul2=0, \n              dropRate=0, optimizer='adam', epochs=10, batch_size=32):\n    model = Sequential()\n    model.add(layers.BatchNormalization())\n    model.add(layers.Dense(512, activation='relu', input_shape=[784]))\n    model.add(layers.Dropout(dropRate))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Dense(256, activation='relu'))\n    model.add(layers.Dropout(dropRate))\n    model.add(layers.Dense(10))\n    \n    # we used linear activation in the last layer, so now we have to set \"from_logits=True\" in the loss function\n    # it's considered to be more numerically stable\n    model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n                  optimizer=optimizer,\n                  metrics=['accuracy'])\n    \n    histModel = model.fit(x=X_train, y=y_train, batch_size=batch_size, epochs=epochs, validation_data=validation_data, verbose=0)\n    \n    TrainAccuracy = histModel.history['accuracy']\n    \n    if validation_data != None:\n        TestAccuracy = histModel.history['val_accuracy']\n        BestScore = max(TestAccuracy)\n    else: \n        TestAccuracy = []\n        BestScore = 0\n    \n    return model, histModel, TrainAccuracy, TestAccuracy, BestScore","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:34:22.564272Z","iopub.execute_input":"2024-08-16T15:34:22.565000Z","iopub.status.idle":"2024-08-16T15:34:22.575303Z","shell.execute_reply.started":"2024-08-16T15:34:22.564972Z","shell.execute_reply":"2024-08-16T15:34:22.574421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, histModel, TrainAccuracy, TestAccuracy, BestScore = train_net(X_train, y_train, validation_data=(X_test, y_test),\n                                                                     nu1=600, nu2=300, nu3=10, activation='relu', \n                                                                     optimizer='adamax', epochs=50, batch_size=64,\n                                                                     regul2=0.001)\nprint('Train accuracy:', max(TrainAccuracy))\nprint('Test accuracy:', BestScore)\nhist_df = pd.DataFrame(histModel.history)\nhist_df.loc[1:,['accuracy', 'val_accuracy']].plot()","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:34:25.091923Z","iopub.execute_input":"2024-08-16T15:34:25.092279Z","iopub.status.idle":"2024-08-16T15:35:26.864693Z","shell.execute_reply.started":"2024-08-16T15:34:25.092245Z","shell.execute_reply":"2024-08-16T15:35:26.863813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df_scaled = scaler.transform(test_df)\n\npredictions = model.predict(test_df_scaled)\npredictions = np.argmax(predictions, axis = 1)\n\nsubmission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv', index_col = False)\nsubmission['Label'] = predictions\nsubmission.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:37:47.122713Z","iopub.execute_input":"2024-08-16T15:37:47.123404Z","iopub.status.idle":"2024-08-16T15:37:49.311582Z","shell.execute_reply.started":"2024-08-16T15:37:47.123372Z","shell.execute_reply":"2024-08-16T15:37:49.310542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_df = pd.DataFrame(X_test)\nX_test_df['prediction'] = np.argmax(model.predict(X_test_df), axis = 1)\nX_test_df['label'] = np.array(y_test)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:38:18.873711Z","iopub.execute_input":"2024-08-16T15:38:18.874322Z","iopub.status.idle":"2024-08-16T15:38:19.391615Z","shell.execute_reply.started":"2024-08-16T15:38:18.874292Z","shell.execute_reply":"2024-08-16T15:38:19.390742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"misclassified = X_test_df.loc[X_test_df['prediction'] != X_test_df['label']]\nmisclassified['label'].value_counts().plot(kind = 'bar')","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:38:21.851511Z","iopub.execute_input":"2024-08-16T15:38:21.851871Z","iopub.status.idle":"2024-08-16T15:38:22.080978Z","shell.execute_reply.started":"2024-08-16T15:38:21.851843Z","shell.execute_reply":"2024-08-16T15:38:22.079993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"misclassified.iloc[:,:-2] = scaler.inverse_transform(misclassified.iloc[:,:-2])\nmisclassified.index = range(1,misclassified.shape[0]+1)\n\nm, n = misclassified.shape\n\nfig, axes = plt.subplots(8,8, figsize=(8,8))\nfig.tight_layout(pad=0.12,rect=[0, 0.03, 1, 0.9])\n\npixels = misclassified.iloc[:,:-2]\nlabels = misclassified.iloc[:,-2:]\n\nfor ax in axes.flat:\n\n    random_index = np.random.randint(m)\n    pixels_reshaped = pixels.iloc[random_index].values.reshape((28,28))\n    \n    ax.imshow(pixels_reshaped, cmap='gray')\n    ax.set_title(str(labels['label'].iloc[random_index]) + ' | ' + str(labels['prediction'].iloc[random_index]))\n    ax.set_axis_off()\n    fig.suptitle(\"Examples of some misclassified digits (true label | predicted label)\", fontsize=14, y = 0.95)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:40:54.770024Z","iopub.execute_input":"2024-08-16T15:40:54.770406Z","iopub.status.idle":"2024-08-16T15:41:00.791275Z","shell.execute_reply.started":"2024-08-16T15:40:54.770375Z","shell.execute_reply":"2024-08-16T15:41:00.790357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:41:51.217789Z","iopub.execute_input":"2024-08-16T15:41:51.218551Z","iopub.status.idle":"2024-08-16T15:41:51.243769Z","shell.execute_reply.started":"2024-08-16T15:41:51.218520Z","shell.execute_reply":"2024-08-16T15:41:51.242881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_test_df.shape)\nmisclassified.shape\n\nlen(misclassified) / len(X_test_df)","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:47:09.509380Z","iopub.execute_input":"2024-08-16T15:47:09.510350Z","iopub.status.idle":"2024-08-16T15:47:09.516694Z","shell.execute_reply.started":"2024-08-16T15:47:09.510317Z","shell.execute_reply":"2024-08-16T15:47:09.515722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"copy_of_test_df = X_test_df\n\ncopy_of_test_df.iloc[:,:-2].head()\n#copy_of_test_df.iloc[:,:-2] = scaler.inverse_transform(copy_of_test_df.iloc[:,:-2])\n\nprint(copy_of_test_df.iloc[:,:-2])\n#img_pixels = imgs_data.iloc[:,:-2]\n#img_labels = imgs_data.iloc[:,-2:]\n\n#plt.imshow(img_pixels[0].values.reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2024-08-16T15:59:33.700398Z","iopub.execute_input":"2024-08-16T15:59:33.701024Z","iopub.status.idle":"2024-08-16T15:59:33.754460Z","shell.execute_reply.started":"2024-08-16T15:59:33.700992Z","shell.execute_reply":"2024-08-16T15:59:33.753523Z"},"trusted":true},"execution_count":null,"outputs":[]}]}