{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"latex_envs":{"LaTeX_envs_menu_present":true,"autoclose":false,"autocomplete":true,"bibliofile":"biblio.bib","cite_by":"apalike","current_citInitial":1,"eqLabelWithNumbers":true,"eqNumInitial":1,"hotkeys":{"equation":"Ctrl-E","itemize":"Ctrl-I"},"labels_anchors":false,"latex_user_defs":false,"report_style_numbering":false,"user_envs_cfg":false},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":75110,"databundleVersionId":8241956,"sourceType":"competition"}],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EMNIST Number & Character Recognition","metadata":{}},{"cell_type":"markdown","source":"Build a classifier for handwritten digit and character recognition\n\nSteps:\n- Environment Check\n- Data Import\n- Train Test Split\n- Image Processing Functions\n- Sighting the data\n- Model Creation\n- Model Training\n- Model Evaluation\n- Submit Kaggle Solution\n- Build Gradio User Interface","metadata":{}},{"cell_type":"markdown","source":"## Environment Check","metadata":{}},{"cell_type":"code","source":"import sys\ntype(sys.path)\nfor path in sys.path:\n    print(path)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:28:11.130677Z","iopub.execute_input":"2024-04-15T11:28:11.130935Z","iopub.status.idle":"2024-04-15T11:28:11.142582Z","shell.execute_reply.started":"2024-04-15T11:28:11.130911Z","shell.execute_reply":"2024-04-15T11:28:11.141679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Show a list of directories in which installed packages are searched for. This gives information about the currently used environment","metadata":{}},{"cell_type":"code","source":"import gc\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom time import time\nimport seaborn as sns\nsns.set_theme(style=\"whitegrid\")\nsns.color_palette(\"viridis\", as_cmap=True)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score\nimport itertools\n\nimport tensorflow as tf\nfrom tensorflow.python.client import device_lib\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom tensorflow.keras.optimizers import RMSprop, Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\nphysical_devices = tf.config.list_physical_devices('GPU')\nprint(physical_devices)","metadata":{"_cell_guid":"f67b9393-8ea1-4e23-b856-2ce149cfe421","_execution_state":"idle","_uuid":"72334cb006d02a4bcfc2a2fe622524eba824c6f8","execution":{"iopub.status.busy":"2024-04-15T11:28:14.940439Z","iopub.execute_input":"2024-04-15T11:28:14.941037Z","iopub.status.idle":"2024-04-15T11:28:28.630317Z","shell.execute_reply.started":"2024-04-15T11:28:14.940994Z","shell.execute_reply":"2024-04-15T11:28:28.629308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_available_gpus():\n    local_device_protos = device_lib.list_local_devices()\n    print(local_device_protos)\n    return [x.name for x in local_device_protos if x.device_type == \"GPU\"]\n\nget_available_gpus()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:28:30.848594Z","iopub.execute_input":"2024-04-15T11:28:30.849523Z","iopub.status.idle":"2024-04-15T11:28:30.958287Z","shell.execute_reply.started":"2024-04-15T11:28:30.849489Z","shell.execute_reply":"2024-04-15T11:28:30.957252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check Compute capabilities. Important for GPU accessability for CNN training later on","metadata":{}},{"cell_type":"code","source":"IMSIZE = 28\nNUM_CLASSES = 62\nBATCH_SIZE = 1024","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:28:37.468265Z","iopub.execute_input":"2024-04-15T11:28:37.468619Z","iopub.status.idle":"2024-04-15T11:28:37.473092Z","shell.execute_reply.started":"2024-04-15T11:28:37.468590Z","shell.execute_reply":"2024-04-15T11:28:37.472004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The images are 28x28 pixels, there are 62 classes 0-9 / a-z / A-Z, Batch Size is 1024.","metadata":{}},{"cell_type":"markdown","source":"## Data Import & Check","metadata":{}},{"cell_type":"code","source":"def plot_label_distribution(labels):\n    plt.figure(figsize=(10,5))\n    chart = sns.countplot(x=labels, palette=\"viridis\")\n    response = chart.set_xticklabels(chart.get_xticklabels(), rotation=45, fontweight='light', fontsize='x-small')\n    \n# Encode labels to one hot vectors\ndef get_topX_onehot(dfbasis, coldata, coldesc=None, numfeatures=0):\n    if coldesc is None:\n        coldesc=coldata\n    toplist = []\n    if numfeatures == 0: #process all features\n        toplist = list(dfbasis[coldata].value_counts().sort_index().index.values)\n    else:\n        toplist = list(dfbasis[coldata].value_counts().sort_index().index.values)[0:numfeatures]\n    #print(toplist)\n    dfnew = dfbasis[[coldata]].copy()\n    for topentry in toplist:\n        featurename = dfbasis[dfbasis[coldata] == topentry][coldesc].value_counts().index[0]\n        colname = str(coldata)+\"_\"+str(featurename)\n        dfnew[colname] = dfbasis[coldata] == topentry\n    if numfeatures > 0: #iuclude \"others\"-column\n        dfnew[coldata+\"_other\"] = ~dfbasis[coldata].isin(toplist)\n    dfnew.drop(coldata, axis=1, inplace=True)\n    return dfnew\n\n# Revert One Hot Encoding \ndef revert_onehot(data_np_onehot, labellist_int):\n    labels = (data_np_onehot*np.asarray(labellist_int)).sum(axis=1)\n    return labels","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:28:38.976682Z","iopub.execute_input":"2024-04-15T11:28:38.977023Z","iopub.status.idle":"2024-04-15T11:28:38.987373Z","shell.execute_reply.started":"2024-04-15T11:28:38.976998Z","shell.execute_reply":"2024-04-15T11:28:38.986369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load training data\ntrain_df = pd.read_csv(\"/kaggle/input/emnistfhj-2024/train.csv\", index_col=\"id_code\")\n# Load test data\ntest_df = pd.read_csv(\"/kaggle/input/emnistfhj-2024/test.csv\", index_col=\"id_code\")\n# Show some lines\ntrain_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:29:03.441005Z","iopub.execute_input":"2024-04-15T11:29:03.441796Z","iopub.status.idle":"2024-04-15T11:30:13.833783Z","shell.execute_reply.started":"2024-04-15T11:29:03.441764Z","shell.execute_reply":"2024-04-15T11:30:13.832780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_classes = train_df[\"label\"].unique()\nall_classes = list(np.sort(all_classes))","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:20.892170Z","iopub.execute_input":"2024-04-15T11:30:20.893038Z","iopub.status.idle":"2024-04-15T11:30:20.905542Z","shell.execute_reply.started":"2024-04-15T11:30:20.893005Z","shell.execute_reply":"2024-04-15T11:30:20.904664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class2ascii = {}\nascii2class = {}\nfor idx, num_ascii in enumerate(all_classes):\n    class2ascii[idx] = num_ascii\n    ascii2class[num_ascii] = idx","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:21.899024Z","iopub.execute_input":"2024-04-15T11:30:21.899856Z","iopub.status.idle":"2024-04-15T11:30:21.904554Z","shell.execute_reply.started":"2024-04-15T11:30:21.899822Z","shell.execute_reply":"2024-04-15T11:30:21.903494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare Training data features and labels\ntrain_y_df = train_df[[\"label\"]]\nprint(f\"Training label shape: {train_y_df.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:22.279575Z","iopub.execute_input":"2024-04-15T11:30:22.280144Z","iopub.status.idle":"2024-04-15T11:30:22.287689Z","shell.execute_reply.started":"2024-04-15T11:30:22.280114Z","shell.execute_reply":"2024-04-15T11:30:22.286650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:24.157144Z","iopub.execute_input":"2024-04-15T11:30:24.157970Z","iopub.status.idle":"2024-04-15T11:30:24.163371Z","shell.execute_reply.started":"2024-04-15T11:30:24.157937Z","shell.execute_reply":"2024-04-15T11:30:24.162431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare Training Features\ntrain_X_df = train_df.drop(labels = [\"label\"], axis = 1) \n# Memory saving & ready for scaling\ntrain_X_df = train_X_df.astype(\"float32\")\n# Normalize values & reshape training data (height = 28px, width = 28px , canal = 1)\ntrain_X_df = train_X_df / 255.0\ntrain_X_np = train_X_df.values.reshape(-1,28,28,1)\nprint(f\"Training data shape: {train_X_np.shape}\")\n# Del unused varaibles from memory\ndel train_df, train_X_df\ngc.collect()\n\n\n# Same for test data\ntest_X_df = test_df\ntest_X_df = test_X_df.astype(\"float32\")\ntest_X_df = test_X_df / 255.0\ntest_X_np = test_X_df.values.reshape(-1,28,28,1)\nprint(f\"Test data shape: {test_X_np.shape}\")\ndel test_X_df\ngc.collect()","metadata":{"_cell_guid":"86570a36-5c20-460a-9dfd-2070548532a7","_execution_state":"idle","_uuid":"1213b979d5ed3e0d13824d17d694c79d2ece92fa","execution":{"iopub.status.busy":"2024-04-15T11:30:24.631392Z","iopub.execute_input":"2024-04-15T11:30:24.631953Z","iopub.status.idle":"2024-04-15T11:30:28.101067Z","shell.execute_reply.started":"2024-04-15T11:30:24.631924Z","shell.execute_reply":"2024-04-15T11:30:28.100035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X_np.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:33.111449Z","iopub.execute_input":"2024-04-15T11:30:33.112131Z","iopub.status.idle":"2024-04-15T11:30:33.117864Z","shell.execute_reply.started":"2024-04-15T11:30:33.112099Z","shell.execute_reply":"2024-04-15T11:30:33.116845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training and Testdata are spit into Feature and Target Matrices/Vectors.\n\nDatatype \"float32\" is used instead of default \"float64\" to save memory.\n\nFeature Matrix is brought into the shape that is needed for training.","metadata":{}},{"cell_type":"code","source":"plot_label_distribution(train_y_df[\"label\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:34.069188Z","iopub.execute_input":"2024-04-15T11:30:34.069618Z","iopub.status.idle":"2024-04-15T11:30:35.491925Z","shell.execute_reply.started":"2024-04-15T11:30:34.069586Z","shell.execute_reply":"2024-04-15T11:30:35.491024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Label distribution is not uniform. However, every class has at least 2k examples after a brief examination.","metadata":{}},{"cell_type":"code","source":"train_y_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:36.768638Z","iopub.execute_input":"2024-04-15T11:30:36.768995Z","iopub.status.idle":"2024-04-15T11:30:36.780552Z","shell.execute_reply.started":"2024-04-15T11:30:36.768965Z","shell.execute_reply":"2024-04-15T11:30:36.779504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The label vector represents each class as its ASCII Code.\n\nFor training we need a one-hot-encoded vector for each sample showing a \"1\" in one of 62 positions","metadata":{}},{"cell_type":"code","source":"train_y_df_onehot = get_topX_onehot(train_y_df, \"label\")\ntrain_y_np = train_y_df_onehot.to_numpy().astype(\"int\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:42.274310Z","iopub.execute_input":"2024-04-15T11:30:42.274664Z","iopub.status.idle":"2024-04-15T11:30:42.673219Z","shell.execute_reply.started":"2024-04-15T11:30:42.274634Z","shell.execute_reply":"2024-04-15T11:30:42.672218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_y_np[0:5,:]","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:43.670174Z","iopub.execute_input":"2024-04-15T11:30:43.670561Z","iopub.status.idle":"2024-04-15T11:30:43.678047Z","shell.execute_reply.started":"2024-04-15T11:30:43.670532Z","shell.execute_reply":"2024-04-15T11:30:43.676994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Test Split & Stratified Split","metadata":{}},{"cell_type":"markdown","source":"https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html\n\nhttps://stackoverflow.com/questions/34842405/parameter-stratify-from-method-train-test-split-scikit-learn","metadata":{}},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\ntrain_X, validate_X, train_y, validate_y = train_test_split(train_X_np, train_y_np, \n                                                            test_size = 0.2, random_state=42)","metadata":{"_cell_guid":"dcd25ebb-d845-4d32-9867-082e352b1396","_execution_state":"idle","_uuid":"b779ac76d8317647db92d5a88b4098d212d72884","execution":{"iopub.status.busy":"2024-04-15T11:30:49.038082Z","iopub.execute_input":"2024-04-15T11:30:49.038453Z","iopub.status.idle":"2024-04-15T11:30:56.612060Z","shell.execute_reply.started":"2024-04-15T11:30:49.038421Z","shell.execute_reply":"2024-04-15T11:30:56.611237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_label_distribution(revert_onehot(train_y, all_classes))","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:56.614012Z","iopub.execute_input":"2024-04-15T11:30:56.614457Z","iopub.status.idle":"2024-04-15T11:30:57.876861Z","shell.execute_reply.started":"2024-04-15T11:30:56.614422Z","shell.execute_reply":"2024-04-15T11:30:57.876013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_label_distribution(revert_onehot(validate_y, all_classes))","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:30:57.878073Z","iopub.execute_input":"2024-04-15T11:30:57.878442Z","iopub.status.idle":"2024-04-15T11:30:59.020071Z","shell.execute_reply.started":"2024-04-15T11:30:57.878408Z","shell.execute_reply":"2024-04-15T11:30:59.019153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Implement a stratified train-test-split here to assure that the percentage of samles in each class is exactly the same in training and validation set","metadata":{}},{"cell_type":"code","source":"# Stratified Split","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:01.714678Z","iopub.execute_input":"2024-04-15T11:31:01.715021Z","iopub.status.idle":"2024-04-15T11:31:01.719236Z","shell.execute_reply.started":"2024-04-15T11:31:01.714994Z","shell.execute_reply":"2024-04-15T11:31:01.718261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_label_distribution(revert_onehot(train_y, label_list))","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:02.163969Z","iopub.execute_input":"2024-04-15T11:31:02.164604Z","iopub.status.idle":"2024-04-15T11:31:02.168649Z","shell.execute_reply.started":"2024-04-15T11:31:02.164571Z","shell.execute_reply":"2024-04-15T11:31:02.167545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_label_distribution(revert_onehot(validate_y, label_list))","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:02.485975Z","iopub.execute_input":"2024-04-15T11:31:02.486284Z","iopub.status.idle":"2024-04-15T11:31:02.490099Z","shell.execute_reply.started":"2024-04-15T11:31:02.486258Z","shell.execute_reply":"2024-04-15T11:31:02.489320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Processing & First sight","metadata":{}},{"cell_type":"code","source":"import albumentations\nimport cv2\n\n#Image Helper Functions\n\ndef show_image(image, figsize=None, title=None):\n    if figsize is not None:\n        fig = plt.figure(figsize=figsize)\n    if image.ndim == 2:\n        plt.imshow(image,cmap='gray')\n    elif image.ndim == 3:\n        if image.shape[2] == 1:\n            image = image[:,:,0]\n            plt.imshow(image,cmap='gray')\n        elif image.shape[2] == 3:\n            plt.imshow(image)\n        else:\n            print(\"Invalid image dimension\")\n    if title is not None:\n        plt.title(title)\n\n        \ndef show_Nimages(imgs, classifications, scale=1):\n    N=len(imgs)\n    fig = plt.figure(figsize=(25/scale, 16/scale))\n    for i in range(N):\n        ax = fig.add_subplot(1, N, i + 1, xticks=[], yticks=[])\n        show_image(imgs[i], title=\"C:{}\".format(classifications[i]))\n\n        \ndef show_Nerrorimages(imgs, predval, trueval, delta=[], cols=None):\n    N=len(imgs)\n    if cols == None:\n        cols = int(np.sqrt(N))\n    rows = (N//cols) + 1\n    fig = plt.figure(figsize=(cols*2, rows*2))\n    \n    for i in range(N):\n        ax = fig.add_subplot(rows, cols, i + 1, xticks=[], yticks=[])\n        if len(delta) == 0:\n            show_image(imgs[i], title=\"P:{} T:{}\".format(predval[i], trueval[i]))\n        else:\n            show_image(imgs[i], title=\"P:{} T:{} d:{}\".format(predval[i], trueval[i], delta[i]))\n            \n\ndef show_Nrandomimages(data_np, label_np, N=10):\n    indices = (np.random.rand(N)*data_np.shape[0]).astype(int)\n    show_Nimages(data_np[indices], label_np[indices])\n\n    \ndef show_Nimages_of_class(data_np, label_np, classification=0, N=10):\n    label_df = pd.DataFrame(data=label_np, columns=[\"label\"])\n    indices = label_df[label_df[\"label\"] == classification].sample(N).index\n    show_Nimages(data_np[indices], label_np[indices])\n\n    \ndef display_generator_images(generator, cols=None):\n    for images, targets in generator:\n        #print(\"Shape X: {}\".format(images.shape))\n        #print(\"Shape y: {}\".format(targets.shape))\n        N = images.shape[0]\n        if cols == None:\n            cols = int(np.sqrt(N))\n        rows = (N//cols) + 1\n        fig = plt.figure(figsize=(cols*2, rows*2))\n        for i, (img, trg) in enumerate(zip(images, targets)):\n            ax = fig.add_subplot(rows, cols, i + 1, xticks=[], yticks=[])\n            show_image(img, title=\"C:{}\".format(np.argmax(trg)))\n        break","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:03.284522Z","iopub.execute_input":"2024-04-15T11:31:03.284879Z","iopub.status.idle":"2024-04-15T11:31:03.701010Z","shell.execute_reply.started":"2024-04-15T11:31:03.284851Z","shell.execute_reply":"2024-04-15T11:31:03.700242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_index = 24790\nshow_image(train_X_np[test_index], title=\"Class = {}\".format(train_y_df[\"label\"][test_index]))","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:04.872163Z","iopub.execute_input":"2024-04-15T11:31:04.872898Z","iopub.status.idle":"2024-04-15T11:31:05.267476Z","shell.execute_reply.started":"2024-04-15T11:31:04.872867Z","shell.execute_reply":"2024-04-15T11:31:05.266464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_Nrandomimages(train_X, revert_onehot(train_y, all_classes), 10)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:07.219073Z","iopub.execute_input":"2024-04-15T11:31:07.219742Z","iopub.status.idle":"2024-04-15T11:31:07.975324Z","shell.execute_reply.started":"2024-04-15T11:31:07.219708Z","shell.execute_reply":"2024-04-15T11:31:07.974432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_Nimages_of_class(train_X, revert_onehot(train_y, all_classes), classification=68)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:08.189329Z","iopub.execute_input":"2024-04-15T11:31:08.190138Z","iopub.status.idle":"2024-04-15T11:31:08.973808Z","shell.execute_reply.started":"2024-04-15T11:31:08.190108Z","shell.execute_reply":"2024-04-15T11:31:08.972925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Neural Network Training","metadata":{}},{"cell_type":"code","source":"reduceLR = ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.5,\n    patience=3,\n    min_lr=1e-5,\n    verbose=1,\n    mode='min'\n)\n\nearlyStopping = EarlyStopping(\n    monitor='val_loss',\n    patience=8,\n    verbose=1,\n    mode='min',\n    restore_best_weights=True\n)\n\ndef build_perceptron_model():\n    model = Sequential()\n    model.add(Flatten(input_shape = (28, 28, 1)))\n    model.add(Dense(128, activation ='relu'))\n    model.add(Dense(NUM_CLASSES, activation = \"softmax\"))\n    return model\n\ndef build_convnet_model_simple():\n    model = Sequential()\n    #define model here\n    return model\n\ndef build_convnet_model_adv():\n    model = Sequential()\n    #define model here\n    return model\n\ndef display_training_results(history):\n    history_df = pd.DataFrame(history.history)\n    f = plt.figure(figsize=(25,5))\n    ax = f.add_subplot(131)\n    ax.plot(history_df[\"loss\"], label=\"loss\")\n    ax.plot(history_df[\"val_loss\"], label = \"val_loss\")\n    ax.legend()\n    ax = f.add_subplot(132)\n    ax.plot(history_df[\"accuracy\"], label=\"acc\")\n    ax.plot(history_df[\"val_accuracy\"], label=\"val_acc\")\n    ax.legend()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:10.196645Z","iopub.execute_input":"2024-04-15T11:31:10.197366Z","iopub.status.idle":"2024-04-15T11:31:10.207542Z","shell.execute_reply.started":"2024-04-15T11:31:10.197332Z","shell.execute_reply":"2024-04-15T11:31:10.206478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multilayer Perceptron Perceptron","metadata":{}},{"cell_type":"code","source":"# Build Model\nmodel_mlp = build_perceptron_model()\n\n# Define optimizer\nmy_optimizer = Adam(learning_rate=1e-3)\n\n# Compile the model\nmodel_mlp.compile(optimizer=my_optimizer , loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\nmodel_mlp.summary()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:11.442785Z","iopub.execute_input":"2024-04-15T11:31:11.443136Z","iopub.status.idle":"2024-04-15T11:31:11.614141Z","shell.execute_reply.started":"2024-04-15T11:31:11.443108Z","shell.execute_reply":"2024-04-15T11:31:11.613295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time()\nhistory = model_mlp.fit(x=train_X, y=train_y, validation_data=(validate_X, validate_y), epochs=20, batch_size=1024)\nduration = time()-start_time\nprint(f\"Model Training in {duration:.4f} secs\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:31:16.913814Z","iopub.execute_input":"2024-04-15T11:31:16.914467Z","iopub.status.idle":"2024-04-15T11:31:58.795826Z","shell.execute_reply.started":"2024-04-15T11:31:16.914431Z","shell.execute_reply":"2024-04-15T11:31:58.794780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_proba = model_mlp.predict(validate_X)\nprediction_num = np.argmax(prediction_proba, axis=1)\nprint(f\"Accuracy Score: {accuracy_score([all_classes[x] for x in list( np.argmax(validate_y, axis = 1))], [all_classes[x] for x in list(prediction_num)]):.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:32:01.457358Z","iopub.execute_input":"2024-04-15T11:32:01.457690Z","iopub.status.idle":"2024-04-15T11:32:10.450612Z","shell.execute_reply.started":"2024-04-15T11:32:01.457665Z","shell.execute_reply":"2024-04-15T11:32:10.449676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image generator for training","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\nfrom sklearn.utils import shuffle\n\nclass My_Generator(Sequence):\n\n    def __init__(self, image_array, labels, batch_size, train=True, augment=False):\n        self.image_array, self.labels = image_array, labels\n        self.batch_size = batch_size\n        self.is_train = train\n        self.is_augment = augment\n            \n        \n        if(self.is_train):\n            self.on_epoch_end() #shuffle\n\n            \n    def on_epoch_end(self):\n        pass    \n            \n\n    def __len__(self):\n        pass\n        # todo\n\n    \n    def __getitem__(self, idx):\n        pass\n        # todo","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:32:12.626087Z","iopub.execute_input":"2024-04-15T11:32:12.626484Z","iopub.status.idle":"2024-04-15T11:32:12.635409Z","shell.execute_reply.started":"2024-04-15T11:32:12.626453Z","shell.execute_reply":"2024-04-15T11:32:12.634601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save and/or Load pretrained models","metadata":{}},{"cell_type":"code","source":"LOAD_MODELS_FROM_DISK = False\nMODEL_PATH = \"../models/\"\n\n#LOAD\nif LOAD_MODELS_FROM_DISK:\n    model_mlp = tf.keras.models.load_model(MODEL_PATH+\"model_mlp.h5\")\n#SAVE\nelse:\n    model_mlp.save(MODEL_PATH+\"model_mlp.h5\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:32:14.147410Z","iopub.execute_input":"2024-04-15T11:32:14.147879Z","iopub.status.idle":"2024-04-15T11:32:14.177920Z","shell.execute_reply.started":"2024-04-15T11:32:14.147847Z","shell.execute_reply":"2024-04-15T11:32:14.177169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate models","metadata":{}},{"cell_type":"code","source":"# Plot confusion matrix \ndef plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n    fig = plt.figure(figsize=(16, 16))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j], horizontalalignment=\"center\", \n                 color=\"white\" if cm[i, j] > thresh else \"black\",\n                 fontsize='xx-small',\n                )\n\n    plt.grid(False)\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    \ndef evaluate_metrics(y_true, y_pred):\n    print('-'*100)\n    print(f\"Accuracy Score: {accuracy_score(y_true, y_pred):.2f}\")\n    print('-'*100)","metadata":{"_cell_guid":"11361e73-8250-4bf5-a353-b0f8ea83e659","_execution_state":"idle","_uuid":"16e161179bf1b51ba66c39b2cead883f1db3a9c7","execution":{"iopub.status.busy":"2024-04-15T11:32:16.779391Z","iopub.execute_input":"2024-04-15T11:32:16.779709Z","iopub.status.idle":"2024-04-15T11:32:16.789704Z","shell.execute_reply.started":"2024-04-15T11:32:16.779685Z","shell.execute_reply":"2024-04-15T11:32:16.788663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process predicted values: onehot --> index --> label\nprediction_proba = model_mlp.predict(validate_X)\nprediction_num = np.argmax(prediction_proba, axis=1)\nprediction_label = [all_classes[x] for x in list(prediction_num)]\n\n# Same for true values\ntrue_num = np.argmax(validate_y, axis = 1)\ntrue_label = [all_classes[x] for x in list(true_num)]\n\n# Confusion Matrix\nconfusion_mtx = confusion_matrix(true_label, prediction_label)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:32:19.366610Z","iopub.execute_input":"2024-04-15T11:32:19.367474Z","iopub.status.idle":"2024-04-15T11:32:26.852824Z","shell.execute_reply.started":"2024-04-15T11:32:19.367440Z","shell.execute_reply":"2024-04-15T11:32:26.851950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_metrics(true_label, prediction_label)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:32:54.016292Z","iopub.execute_input":"2024-04-15T11:32:54.016623Z","iopub.status.idle":"2024-04-15T11:32:54.076165Z","shell.execute_reply.started":"2024-04-15T11:32:54.016599Z","shell.execute_reply":"2024-04-15T11:32:54.075228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confusion_matrix(confusion_mtx, classes = [chr(x) for x in all_classes]) ","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:32:55.124517Z","iopub.execute_input":"2024-04-15T11:32:55.124836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classification Errors","metadata":{}},{"cell_type":"code","source":"prediction_label_np = np.array(prediction_label)\ntrue_label_np = np.array(true_label)\n# Errors are difference between predicted labels and true labels\nerrors = (prediction_label_np - true_label_np != 0)\n# true_label\n# prediction_label\nerror_X = validate_X[errors]\nerror_y_true = true_label_np[errors]\nerror_y_true_index = np.array([all_classes.index(x) for x in list(error_y_true)])\nerror_y_pred = prediction_label_np[errors]\n\nindices = (np.random.rand(100)*error_X.shape[0]).astype(int)\nshow_Nerrorimages(error_X[indices], \n                  predval=error_y_pred[indices], \n                  trueval=error_y_true[indices])\n\n# Probabilities of the wrong predicted numbers\nerror_y_proba = prediction_proba[errors]\nprob_wrong_prediction = np.max(error_y_proba,axis = 1)\n\n# Predicted probabilities of the true values in the error set\nprob_true_prediction = np.diagonal(np.take(error_y_proba, error_y_true_index, axis=1))\n# Difference between the probability of the predicted label and the true label\ndelta_errors = prob_wrong_prediction - prob_true_prediction\n\n# Sorted list of the delta prob errors\ndelta_errors_sorted = np.argsort(delta_errors)\n\nmost_important_errors = delta_errors_sorted[-100:]\n\nshow_Nerrorimages(error_X[most_important_errors],\n                  predval=error_y_pred[most_important_errors], \n                  trueval=error_y_true[most_important_errors],\n                  delta=delta_errors[most_important_errors])","metadata":{"_cell_guid":"7b0f31b8-c18b-4529-b0d8-eb4c31e30bbf","_execution_state":"idle","_uuid":"e7a3d6449b499a29db224e42e950f21ca1ec4e36","execution":{"iopub.status.busy":"2024-04-15T11:33:08.661326Z","iopub.execute_input":"2024-04-15T11:33:08.662025Z","iopub.status.idle":"2024-04-15T11:33:22.932108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Some Chars","metadata":{}},{"cell_type":"code","source":"def classify_image(img, mymodel, mylabellist):\n    test_image_4dim = np.expand_dims(test_image, axis=0)\n    result_proba = mymodel.predict(test_image_4dim)\n    result_index = np.argmax(result_proba, axis = 1)[0]\n    result_class = mylabellist[result_index]\n    return result_class","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:33:22.933827Z","iopub.execute_input":"2024-04-15T11:33:22.934101Z","iopub.status.idle":"2024-04-15T11:33:22.939772Z","shell.execute_reply.started":"2024-04-15T11:33:22.934077Z","shell.execute_reply":"2024-04-15T11:33:22.938810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_INDEX = 111\n\ntest_image = test_X_np[TEST_INDEX] # test images, 28x28 pixels, 1 layer, values in [0,1]\nprint(f\"Image shape: {test_image.shape}\")\nshow_image(test_image, figsize=(3,3), title = \"Test Image\")\nresult_class = classify_image(test_image, model_mlp, all_classes)\nprint(f\"Result class: {result_class} is character '{chr(result_class)}'\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:33:22.940889Z","iopub.execute_input":"2024-04-15T11:33:22.941180Z","iopub.status.idle":"2024-04-15T11:33:23.492621Z","shell.execute_reply.started":"2024-04-15T11:33:22.941157Z","shell.execute_reply":"2024-04-15T11:33:23.491694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Kaggle Submission","metadata":{}},{"cell_type":"code","source":"# predict results\nresults = model_mlp.predict(test_X_np)\nresults = np.argmax(results,axis = 1)\nresults_df = pd.DataFrame(results, columns=[\"label\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:33:24.840817Z","iopub.execute_input":"2024-04-15T11:33:24.841577Z","iopub.status.idle":"2024-04-15T11:33:35.426627Z","shell.execute_reply.started":"2024-04-15T11:33:24.841544Z","shell.execute_reply":"2024-04-15T11:33:35.425617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df[\"label_ascii\"] = results_df[\"label\"].apply(lambda x: all_classes[x])\nsample_submission = pd.read_csv(\"/kaggle/input/emnistfhj-2024/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:34:01.888683Z","iopub.execute_input":"2024-04-15T11:34:01.889324Z","iopub.status.idle":"2024-04-15T11:34:02.556416Z","shell.execute_reply.started":"2024-04-15T11:34:01.889294Z","shell.execute_reply":"2024-04-15T11:34:02.555340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission[\"label\"] = results_df[\"label_ascii\"]","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:34:03.672689Z","iopub.execute_input":"2024-04-15T11:34:03.673310Z","iopub.status.idle":"2024-04-15T11:34:03.678236Z","shell.execute_reply.started":"2024-04-15T11:34:03.673279Z","shell.execute_reply":"2024-04-15T11:34:03.677257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(\"/kaggle/working/emnist_mlp_zero_baseline_submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:34:04.505941Z","iopub.execute_input":"2024-04-15T11:34:04.506321Z","iopub.status.idle":"2024-04-15T11:34:04.730067Z","shell.execute_reply.started":"2024-04-15T11:34:04.506291Z","shell.execute_reply":"2024-04-15T11:34:04.729302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:34:05.907106Z","iopub.execute_input":"2024-04-15T11:34:05.907479Z","iopub.status.idle":"2024-04-15T11:34:05.920632Z","shell.execute_reply.started":"2024-04-15T11:34:05.907448Z","shell.execute_reply":"2024-04-15T11:34:05.919531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Gradio User Interface","metadata":{}},{"cell_type":"code","source":"import gradio as gr\nimport cv2\n\nMODEL_PATH = \"../models/\"\nmodel = tf.keras.models.load_model(MODEL_PATH+\"model_mlp.h5\")\n\ndef predict_character(sketch):\n    img = cv2.resize(sketch, (28, 28))\n    img = img.reshape(1, 28, 28, 1)\n    img = img.astype(\"float\")/255\n    \n    result_proba = model_mlp.predict(img)\n    result_index = np.argmax(result_proba, axis = 1)[0]\n    result_class = all_classes[result_index]\n    \n    outstring = \"\"\n    ranking = result_proba.argsort()[0]\n    for idx in ranking[::-1][0:5]:\n        outstring = outstring + f\"Character '{chr(all_classes[idx])}': Probability: {result_proba[0][idx]:.4f}\\n\"\n    \n    return f\"'{chr(result_class)}' (#{result_class})\", outstring\n    \niface = gr.Interface(fn=predict_character, inputs='sketchpad', outputs=['label', 'text']).launch(share=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T11:34:09.595447Z","iopub.execute_input":"2024-04-15T11:34:09.596364Z","iopub.status.idle":"2024-04-15T11:34:09.644662Z","shell.execute_reply.started":"2024-04-15T11:34:09.596327Z","shell.execute_reply":"2024-04-15T11:34:09.643601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}