{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n
\n\n

MNIST: Digits Recognizer (CNNs)\n

\n
","metadata":{}},{"cell_type":"markdown","source":"1. [Introduction](#introduction)\n1. [Data Collection](#datacollection)\n1. [Data Analysis](#dataanalysis)\n 1. [Explore Dataset](#exploredataset)\n 1. [Explore Target Variable](#exploretargetvariable)\n 1. [Data Visualization](#datavisualization)\n1. [Data Preprocessing](#datapreprocessing)\n 1. [Deskew](#dpdeskew)\n 1. [Data Augmentation](#dpdataaugmentation)\n1. [Model Training and Evaluation](#trainandevaluate)\n 1. [Multilayer Perceptron](#mlp)\n 1. [LeNet-5](#lenet5)\n 1. [AlexNet](#alexnet)\n 1. [ResNet](#resnet)\n 1. [MobileNet (Transfer Learning)](#mobilenet)\n 1. [Ensembling](#ensembling)\n1. [Submission](#submission)\n1. [Conclusion](#conclusion)","metadata":{}},{"cell_type":"markdown","source":"# Introduction ","metadata":{}},{"cell_type":"markdown","source":"In my previous notebook in Mnist competition I used standard Sklearn classifiers and achieved 99.21% performance. In this educational notebook I will try to achieve better accuracy using neural networks. In particular, I will test classic for current time computer vision architectures such as AlexNet, LeNet, ResNet and so on.","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.simplefilter(action='ignore', category=(RuntimeWarning, FutureWarning, UserWarning))\n\nimport time\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport itertools\nimport functools\nimport re\nimport copy\nfrom functools import partial\nimport tensorflow as tf\nimport keras_tuner as kt\nimport keras\nfrom keras import layers, Model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D, Reshape, MaxPool2D, AveragePooling2D, GlobalAvgPool2D, Input, BatchNormalization, concatenate, SeparableConv2D, Average, SpatialDropout2D\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom keras.optimizers import SGD, Adam, Nadam, RMSprop\nfrom keras.utils import to_categorical, plot_model\nfrom keras.losses import categorical_crossentropy, SparseCategoricalCrossentropy\nfrom keras.metrics import categorical_accuracy\nfrom IPython.display import Markdown, display\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report, f1_score, precision_score, recall_score, make_scorer,roc_curve, precision_recall_curve\nfrom sklearn.model_selection import train_test_split, learning_curve\nfrom scipy.stats import randint\nfrom matplotlib.colors import ListedColormap\n\nfrom scipy.ndimage import interpolation\n\n# to make this notebook's output stable across runs\nnp.random.seed(42)\ntf.random.set_seed(42)\n\nON_KAGGLE = os.getenv(\"KAGGLE_KERNEL_RUN_TYPE\") is not None\n\nverbose_tuning = 0\n\ndef download_from_kaggle(competition):\n command = \"kaggle competitions download -c \"\n filepath = Path(\"data/\" + competition + \".zip\")\n\n if not filepath.is_file():\n subprocess.run((command + competition).split())\n Path(\"data\").mkdir(parents=True, exist_ok=True)\n shutil.unpack_archive(competition + \".zip\", \"data\")\n shutil.move(competition + \".zip\", \"data\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T16:04:17.562549Z","iopub.execute_input":"2023-05-23T16:04:17.563463Z","iopub.status.idle":"2023-05-23T16:04:17.582593Z","shell.execute_reply.started":"2023-05-23T16:04:17.563424Z","shell.execute_reply":"2023-05-23T16:04:17.581210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Collection ","metadata":{}},{"cell_type":"code","source":"competition = \"digit-recognizer\"\nif not ON_KAGGLE:\n download_from_kaggle(competition)\n train_path = \"data/train.csv\"\n test_path = \"data/test.csv\"\nelse:\n train_path = f\"/kaggle/input/{competition}/train.csv\"\n test_path = f\"/kaggle/input/{competition}/test.csv\"\n \ntrain_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:17.592637Z","iopub.execute_input":"2023-05-23T16:04:17.592959Z","iopub.status.idle":"2023-05-23T16:04:23.263949Z","shell.execute_reply.started":"2023-05-23T16:04:17.592937Z","shell.execute_reply":"2023-05-23T16:04:23.262986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis ","metadata":{}},{"cell_type":"markdown","source":"## Explore Dataset ","metadata":{}},{"cell_type":"markdown","source":"Explore the MNIST dataset by analyzing its distribution and visualizing some of its samples.","metadata":{}},{"cell_type":"code","source":"print(f'Train dataset shape:\\t {train_df.shape}',\n f'\\nTest dataset shape:\\t {test_df.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.266096Z","iopub.execute_input":"2023-05-23T16:04:23.266798Z","iopub.status.idle":"2023-05-23T16:04:23.272803Z","shell.execute_reply.started":"2023-05-23T16:04:23.266765Z","shell.execute_reply":"2023-05-23T16:04:23.271722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.274398Z","iopub.execute_input":"2023-05-23T16:04:23.274792Z","iopub.status.idle":"2023-05-23T16:04:23.307336Z","shell.execute_reply.started":"2023-05-23T16:04:23.274763Z","shell.execute_reply":"2023-05-23T16:04:23.306529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.309699Z","iopub.execute_input":"2023-05-23T16:04:23.310019Z","iopub.status.idle":"2023-05-23T16:04:23.326207Z","shell.execute_reply.started":"2023-05-23T16:04:23.309974Z","shell.execute_reply":"2023-05-23T16:04:23.325193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for null and missing values.","metadata":{}},{"cell_type":"code","source":"print('Train dataset, missing values:', train_df.isna().sum().sum())\nprint('Test dataset, missing values:', test_df.isna().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.327794Z","iopub.execute_input":"2023-05-23T16:04:23.329041Z","iopub.status.idle":"2023-05-23T16:04:23.375416Z","shell.execute_reply.started":"2023-05-23T16:04:23.329006Z","shell.execute_reply":"2023-05-23T16:04:23.374397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no missing values in the training and test datasets. So there are no corrupted images.","metadata":{}},{"cell_type":"markdown","source":"### Explore Target Variable ","metadata":{}},{"cell_type":"code","source":"def countplot_target_var(df, label, title):\n ax = sns.countplot(x=label, data=df)\n ax.figure.suptitle(title)\n for container in ax.containers:\n ax.bar_label(container)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T16:04:23.376546Z","iopub.execute_input":"2023-05-23T16:04:23.377914Z","iopub.status.idle":"2023-05-23T16:04:23.383582Z","shell.execute_reply.started":"2023-05-23T16:04:23.377880Z","shell.execute_reply":"2023-05-23T16:04:23.382595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countplot_target_var(train_df, 'label', 'Digits count')","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.384827Z","iopub.execute_input":"2023-05-23T16:04:23.385393Z","iopub.status.idle":"2023-05-23T16:04:23.718474Z","shell.execute_reply.started":"2023-05-23T16:04:23.385362Z","shell.execute_reply":"2023-05-23T16:04:23.717408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks almost balanced.","metadata":{}},{"cell_type":"markdown","source":"### Data Visualization ","metadata":{}},{"cell_type":"markdown","source":"Create function to plot digits.","metadata":{}},{"cell_type":"code","source":"def plot_digits(instances, actual_labels, predicted_labels, images_per_row=10, figsize=(8,8), ticks=[], **options):\n size = 28\n images_per_row = min(len(instances), images_per_row)\n n_rows = (len(instances) - 1) // images_per_row + 1\n n_empty = n_rows * images_per_row - len(instances)\n padded_instances = np.concatenate([instances, np.zeros((n_empty, size * size))], axis=0)\n fig = plt.figure(figsize=figsize)\n for i in range(len(padded_instances)):\n ax = fig.add_subplot(n_rows, images_per_row, i + 1)\n grid_data = padded_instances[i].reshape(28,28)\n plt.imshow(grid_data, cmap = mpl.cm.binary, interpolation=\"nearest\")\n actual_title = f'Actual: {actual_labels[i]}' if len(actual_labels) else ''\n predicted_title = f'\\nPredicted: {predicted_labels[i]}' if len(predicted_labels) else ''\n ax.set_title(f'{actual_title}{predicted_title}')\n if len(ticks) == 0:\n ax.get_xaxis().set_ticks([])\n ax.get_yaxis().set_ticks([])\n else:\n ax.get_xaxis().set_ticks(ticks)\n ax.get_yaxis().set_ticks(ticks)\n ax.grid(True)\n# ax.patch.set_linewidth('1')\n ax.spines['bottom'].set_color('lightgray')\n ax.spines['top'].set_color('lightgray') \n ax.spines['right'].set_color('lightgray')\n ax.spines['left'].set_color('lightgray')\n fig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.720220Z","iopub.execute_input":"2023-05-23T16:04:23.720907Z","iopub.status.idle":"2023-05-23T16:04:23.733532Z","shell.execute_reply.started":"2023-05-23T16:04:23.720870Z","shell.execute_reply":"2023-05-23T16:04:23.732444Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_index = 20\nexample_image = train_df.iloc[example_index, 1:785]\nexample_label = train_df.iloc[example_index, 0]\nplot_digits([example_image], [example_label], [], images_per_row=1, figsize=(3,3), ticks=np.arange(0, 29, 4))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:23.735213Z","iopub.execute_input":"2023-05-23T16:04:23.735644Z","iopub.status.idle":"2023-05-23T16:04:24.128540Z","shell.execute_reply.started":"2023-05-23T16:04:23.735608Z","shell.execute_reply":"2023-05-23T16:04:24.126538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot multiple random digits.\n","metadata":{}},{"cell_type":"code","source":"exmaple_indexes = randint.rvs(0, len(train_df)-1, size=49)\nexample_images = train_df.iloc[exmaple_indexes, 1:785].to_numpy()\nexample_labels = train_df.iloc[exmaple_indexes, 0].to_numpy()\nplot_digits(example_images, example_labels, [], images_per_row=7)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:24.136597Z","iopub.execute_input":"2023-05-23T16:04:24.137061Z","iopub.status.idle":"2023-05-23T16:04:34.038140Z","shell.execute_reply.started":"2023-05-23T16:04:24.137019Z","shell.execute_reply":"2023-05-23T16:04:34.037247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing ","metadata":{}},{"cell_type":"markdown","source":"## Deskew ","metadata":{}},{"cell_type":"markdown","source":"Handwritten digits are skewed. Deskewing images works well for classical models like SVC, so try it, and for neural networks.","metadata":{}},{"cell_type":"code","source":"def moments(image):\n c0,c1 = np.mgrid[:image.shape[0],:image.shape[1]] # A trick in numPy to create a mesh grid\n totalImage = np.sum(image) #sum of pixels\n m0 = np.sum(c0*image)/totalImage #mu_x\n m1 = np.sum(c1*image)/totalImage #mu_y\n m00 = np.sum((c0-m0)**2*image)/totalImage #var(x)\n m11 = np.sum((c1-m1)**2*image)/totalImage #var(y)\n m01 = np.sum((c0-m0)*(c1-m1)*image)/totalImage #covariance(x,y)\n mu_vector = np.array([m0,m1]) # Notice that these are \\mu_x, \\mu_y respectively\n covariance_matrix = np.array([[m00,m01],[m01,m11]]) # Do you see a similarity between the covariance matrix\n return mu_vector, covariance_matrix\n\ndef deskew(image):\n c,v = moments(image)\n alpha = v[0,1]/v[0,0]\n affine = np.array([[1,0],[alpha,1]])\n ocenter = np.array(image.shape)/2.0\n offset = c-np.dot(affine,ocenter)\n return interpolation.affine_transform(image,affine,offset=offset)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:34.039175Z","iopub.execute_input":"2023-05-23T16:04:34.039493Z","iopub.status.idle":"2023-05-23T16:04:34.049334Z","shell.execute_reply.started":"2023-05-23T16:04:34.039464Z","shell.execute_reply":"2023-05-23T16:04:34.048088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_full_deskewed = np.apply_along_axis(lambda x: deskew(x.reshape(28,28)), axis=1, \n arr=train_df.drop(\"label\", axis=1).astype(np.uint8).to_numpy()).astype('float32')\nX_test_deskewed = np.apply_along_axis(lambda x: deskew(x.reshape(28,28)), axis=1, \n arr=test_df.astype(np.uint8).to_numpy()).astype('float32')\nprint(f\"X_train_deskewed shape: {X_full_deskewed.shape}\")\nprint(f\"X_test_deskewed shape: {X_test_deskewed.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:04:34.051294Z","iopub.execute_input":"2023-05-23T16:04:34.051928Z","iopub.status.idle":"2023-05-23T16:05:01.094241Z","shell.execute_reply.started":"2023-05-23T16:04:34.051897Z","shell.execute_reply":"2023-05-23T16:05:01.093289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reshape and Normalization","metadata":{}},{"cell_type":"markdown","source":"Add channel axis and normalize.","metadata":{}},{"cell_type":"code","source":"X_full = np.reshape(X_full_deskewed, (X_full_deskewed.shape[:3]) + (1,)) / 255\nX_test = np.reshape(X_test_deskewed, (X_test_deskewed.shape[:3]) + (1,)) / 255\nprint(f'X_full shape. \\t\\tX: {X_full.shape}',\n f'\\nX_test shape. \\t\\tX: {X_test.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:01.095674Z","iopub.execute_input":"2023-05-23T16:05:01.096239Z","iopub.status.idle":"2023-05-23T16:05:01.167065Z","shell.execute_reply.started":"2023-05-23T16:05:01.096204Z","shell.execute_reply":"2023-05-23T16:05:01.165857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One-Hot Encoding","metadata":{}},{"cell_type":"code","source":"y_full_oh = to_categorical(train_df[\"label\"], num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:01.168694Z","iopub.execute_input":"2023-05-23T16:05:01.169095Z","iopub.status.idle":"2023-05-23T16:05:01.174730Z","shell.execute_reply.started":"2023-05-23T16:05:01.169060Z","shell.execute_reply":"2023-05-23T16:05:01.173855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Splitting dataset","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_full, y_full_oh, \n test_size=0.15, stratify = y_full_oh)\nprint(f'Train shape. \\t\\tX: {X_train.shape}, y: {y_train.shape}',\n f'\\nValidation shape. \\tX: {X_val.shape}, y: {y_val.shape}',\n f'\\nTest shape. \\t\\tX: {X_test.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:01.176112Z","iopub.execute_input":"2023-05-23T16:05:01.176651Z","iopub.status.idle":"2023-05-23T16:05:01.814190Z","shell.execute_reply.started":"2023-05-23T16:05:01.176620Z","shell.execute_reply":"2023-05-23T16:05:01.813210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Resizing to 32x32x3","metadata":{}},{"cell_type":"markdown","source":"Further down I will use the Keras application and it needs images of size 32x32 with 3 channels.","metadata":{}},{"cell_type":"code","source":"def resize_28_1_to_32_3(X):\n X_32 = np.pad(X, ((0,0), (2,2), (2,2), (0,0)), mode='constant')\n X_32_2D = np.reshape(X_32, (X_32.shape[:3]))\n X_32_3 = np.stack((X_32_2D,)*3, axis=-1)\n return X_32_3","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:01.815410Z","iopub.execute_input":"2023-05-23T16:05:01.817627Z","iopub.status.idle":"2023-05-23T16:05:01.823721Z","shell.execute_reply.started":"2023-05-23T16:05:01.817593Z","shell.execute_reply":"2023-05-23T16:05:01.822633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_32_3 = resize_28_1_to_32_3(X_train)\nX_val_32_3 = resize_28_1_to_32_3(X_val)\nX_test_32_3 = resize_28_1_to_32_3(X_test)\nprint(f'Train shape. \\t\\tX: {X_train_32_3.shape}', \n f'\\nValidation shape. \\tX: {X_val_32_3.shape}',\n f'\\nTest shape. \\t\\tX: {X_test_32_3.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:01.825398Z","iopub.execute_input":"2023-05-23T16:05:01.825807Z","iopub.status.idle":"2023-05-23T16:05:02.396697Z","shell.execute_reply.started":"2023-05-23T16:05:01.825776Z","shell.execute_reply":"2023-05-23T16:05:02.395700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data augmentation ","metadata":{}},{"cell_type":"markdown","source":"Create an image generator with random rotation, translation and zoom. We will use it for model training.","metadata":{}},{"cell_type":"code","source":"img_gen = tf.keras.preprocessing.image.ImageDataGenerator(\n rotation_range=10,\n width_shift_range=0.1,\n height_shift_range=0.1,\n zoom_range=0.1,\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:02.398240Z","iopub.execute_input":"2023-05-23T16:05:02.398570Z","iopub.status.idle":"2023-05-23T16:05:02.405650Z","shell.execute_reply.started":"2023-05-23T16:05:02.398538Z","shell.execute_reply":"2023-05-23T16:05:02.404797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training and Evaluation ","metadata":{}},{"cell_type":"markdown","source":"Creating classes for reporting. Instead of using separate functions, I also created EstimatorReport and GridSearchReport classes for reporting functionality.","metadata":{}},{"cell_type":"code","source":"import scikitplot as skplt\n\nclass EstimatorReport():\n \"\"\"\n A class used for generating a report on an estimator's performance \n\n Attributes\n ----------\n X_val : np.ndarray\n Validation dataset features.\n y_val : np.ndarray\n Validation dataset labels.\n X_train : np.ndarray\n Training dataset features.\n y_train : np.ndarray\n Training dataset labels.\n estimator : estimator object\n Scikit-learn estimator object.\n estimator_name : str\n Name of the estimator.\n plot_micro : bool\n Whether micro average ROC curve should be plotted.\n plot_macro : bool\n Whether macro average ROC curve should be plotted.\n metrics_calculated : bool\n Whether the performance metrics have been calculated.\n\n Methods\n -------\n calculate_metrics()\n Calculates accuracy, confusion matrix, classification report and predicted probabilities\n based on the validation set and sets metrics_calculated flag to True.\n plot_curves(figsize=(18, 4))\n Plots ROC and PR curves and learning curve based on the validation set.\n display_report(figsize=(10, 5))\n Displays a report on estimator's performance based on validation set.\n __plot_confusion_matrix(ax)\n Plots confusion matrix.\n __plot_classification_report(ax)\n Plots classification report.\n \"\"\"\n def __init__(\n self,\n estimator,\n X_val: np.ndarray,\n y_val: np.ndarray,\n X_train: np.ndarray,\n y_train: np.ndarray,\n estimator_name: str,\n argmax = False,\n classes_count = None\n ) -> None:\n self.X_val = X_val\n self.y_val = y_val\n self.X_train = X_train\n self.y_train = y_train\n self.estimator = estimator\n self.estimator_name = estimator_name\n self.plot_micro = False\n self.plot_macro = False\n self.metrics_calculated = False\n self.argmax = argmax\n self.classes_count = classes_count\n \n def calculate_metrics(self) -> None:\n self.y_val_predicted = self.estimator.predict(self.X_val)\n if self.argmax:\n self.y_val_probas = self.estimator.predict(self.X_val)\n self.y_val_predicted = self.y_val_probas.argmax(axis=-1)\n else:\n self.y_val_predicted = self.estimator.predict(self.X_val)\n try:\n self.y_val_probas = self.estimator.predict_proba(self.X_val)\n except:\n self.y_val_probas = self.estimator.decision_function(self.X_val)\n self.accuracy = accuracy_score(self.y_val, self.y_val_predicted)\n self.conf_mx = confusion_matrix(self.y_val, self.y_val_predicted)\n self.classif_report = classification_report(\n self.y_val, self.y_val_predicted, output_dict=True\n )\n self.metrics_calculated = True\n \n def plot_curves(self, figsize=(18, 4)) -> None:\n if not self.metrics_calculated:\n self.calculate_metrics()\n fig, axes = plt.subplots( \n 1, 3, figsize=figsize, sharex=False, \n gridspec_kw={'width_ratios': [1, 1, 1]}, dpi=100\n )\n skplt.metrics.plot_roc(\n y_true=self.y_val, y_probas=self.y_val_probas, ax=axes[0], \n title='ROC curve', plot_micro=self.plot_micro, plot_macro=self.plot_macro\n )\n skplt.metrics.plot_precision_recall(\n y_true=self.y_val, y_probas=self.y_val_probas, ax=axes[1], \n title='PR curve', plot_micro=self.plot_micro\n )\n skplt.estimators.plot_learning_curve(\n self.estimator, self.X_train, self.y_train, ax=axes[2]\n )\n axes[0].grid()\n axes[1].grid()\n \n def display_report(self, figsize=(10, 5)) -> None:\n if not self.metrics_calculated:\n self.calculate_metrics()\n display(Markdown(f'### Evaluation on the Validation Set (estimator: {self.estimator_name})'))\n display(Markdown('Accuracy: {:.4f}%'.format(self.accuracy*100)))\n fig, (ax0, ax1) = plt.subplots(1, 2, figsize=figsize, sharex=False, \n gridspec_kw={'width_ratios': [1, 2.2]}, dpi=100)\n self.__plot_classification_report(ax0)\n self.__plot_confusion_matrix(ax1)\n fig.tight_layout()\n plt.show()\n \n def __plot_confusion_matrix(self, ax) -> None:\n norm_conf_mx = self.conf_mx.astype('float') / self.conf_mx.sum(axis=1, keepdims=True)\n np.fill_diagonal(norm_conf_mx, 0)\n sns.heatmap(\n norm_conf_mx, ax=ax, \n annot=False, cmap=plt.cm.gray, cbar=False\n )\n plt.title('Confusion matrix')\n thresh = norm_conf_mx.max() / 2\n for i, j in itertools.product(range(self.conf_mx.shape[0]), range(self.conf_mx.shape[1])):\n plt.text(j+0.5, i+0.6, self.conf_mx[i, j],\n ha=\"center\",\n va='baseline',\n color=\"white\" if norm_conf_mx[i, j] < thresh else \"black\")\n plt.tight_layout()\n plt.ylabel('True label')\n plt.xlabel('Predicted label')\n \n def __plot_classification_report(self, ax) -> None:\n clf_report_df = pd.DataFrame(self.classif_report ).iloc[:-1, :].T\n classes_count = self.classes_count if (self.classes_count != None) else len(self.estimator.classes_)\n mask_classes = np.zeros_like(clf_report_df, dtype=bool)\n mask_classes[classes_count:, :] = True\n mask_metrics = np.zeros_like(clf_report_df, dtype=bool)\n mask_metrics[:classes_count, :] = True\n cmap_white = ListedColormap(['white'])\n sns.heatmap(\n clf_report_df, ax=ax, mask=mask_classes, \n annot=True, cmap='viridis', fmt='.6f', cbar=False\n )\n sns.heatmap(\n clf_report_df, ax=ax, mask=mask_metrics, \n annot=True, cmap=cmap_white, fmt='.6f', cbar=False\n )\n ax.set_title(\"Classification report\", fontsize=10, pad=10)\n ax.xaxis.tick_top()\n ax.xaxis.set_label_position('top')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T16:05:02.407429Z","iopub.execute_input":"2023-05-23T16:05:02.407841Z","iopub.status.idle":"2023-05-23T16:05:02.526472Z","shell.execute_reply.started":"2023-05-23T16:05:02.407811Z","shell.execute_reply":"2023-05-23T16:05:02.525600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history):\n plt.figure(figsize=(8, 5))\n for key, style in zip(history.history, [\"r--\", \"r--.\", \"b-\", \"b-*\"]):\n epochs = np.array(history.epoch) + (0 if key.startswith(\"val_\") else -0.5)\n plt.plot(epochs, history.history[key], style, label=key)\n plt.xlabel(\"Epoch\")\n plt.ylabel(\"Validation Loss and Accuracy\", color='b')\n plt.axis([-0.5, 29, 0., 1])\n plt.legend(loc=\"lower left\")\n plt.grid()\n ax2 = plt.gca().twinx()\n ax2.plot(history.epoch, history.history[\"lr\"], \"r^-\")\n ax2.set_ylabel('Learning Rate', color='r')\n ax2.tick_params('y', colors='r')\n# plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T16:05:02.527911Z","iopub.execute_input":"2023-05-23T16:05:02.528277Z","iopub.status.idle":"2023-05-23T16:05:02.538325Z","shell.execute_reply.started":"2023-05-23T16:05:02.528246Z","shell.execute_reply":"2023-05-23T16:05:02.537412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def grid_search_report(grid_search, res_num=10):\n tuner_report_df = pd.DataFrame()\n trials = grid_search.oracle.get_best_trials(num_trials=15)\n for trial in trials:\n tuner_report_df = tuner_report_df.append(\n trial.hyperparameters.get_config()[\"values\"] | {\"Score\": trial.score}, \n ignore_index=True)\n display(tuner_report_df.head(res_num))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T16:05:02.539778Z","iopub.execute_input":"2023-05-23T16:05:02.540206Z","iopub.status.idle":"2023-05-23T16:05:02.549368Z","shell.execute_reply.started":"2023-05-23T16:05:02.540156Z","shell.execute_reply":"2023-05-23T16:05:02.548499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_errors(estimator, X, y, size, per_row, figsize=(12,10)):\n y_predicted = estimator.predict(X).argmax(axis=-1)\n errors = ((y_predicted - y) != 0)\n X_errors = X[errors]\n y_errors = y[errors]\n y_errors_predicted = y_predicted[errors]\n exmaple_indexes = randint.rvs(0, len(X_errors)-1, size=size)\n example_images = X_errors[exmaple_indexes]\n example_labels = y_errors[exmaple_indexes]\n example_labels_pred = y_errors_predicted[exmaple_indexes]\n n, a, b, c = example_images.shape\n example_images = example_images.reshape((n, a*b*c))\n plot_digits(example_images, example_labels, example_labels_pred, images_per_row=per_row, figsize=figsize)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T17:24:57.697742Z","iopub.execute_input":"2023-05-23T17:24:57.698113Z","iopub.status.idle":"2023-05-23T17:24:57.707340Z","shell.execute_reply.started":"2023-05-23T17:24:57.698087Z","shell.execute_reply":"2023-05-23T17:24:57.705827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_to_results(model_name, acc, results_df):\n row = {\n 'model': model_name,\n 'accuracy': acc\n }\n if model_name in results_df['model'].values:\n row_index = results_df.loc[results_df['model'] == model_name].index[0]\n results_df.loc[row_index, 'accuracy'] = row['accuracy']\n else:\n results_df = results_df.append(row, ignore_index=True)\n return results_df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-23T16:20:45.601693Z","iopub.execute_input":"2023-05-23T16:20:45.602194Z","iopub.status.idle":"2023-05-23T16:20:45.613569Z","shell.execute_reply.started":"2023-05-23T16:20:45.602158Z","shell.execute_reply":"2023-05-23T16:20:45.612671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create variables for early models training stopping and learning rate reducing (factor 0.5 is get from the fine tuning).","metadata":{}},{"cell_type":"code","source":"early_stopping = EarlyStopping(\n monitor='val_categorical_accuracy', \n patience=10,\n verbose=1,\n restore_best_weights = True)\n\nreduce_lr_on_plateau = ReduceLROnPlateau(monitor='val_categorical_accuracy', \n patience=3, verbose=1, factor=0.3)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:02.571402Z","iopub.execute_input":"2023-05-23T16:05:02.571744Z","iopub.status.idle":"2023-05-23T16:05:02.581812Z","shell.execute_reply.started":"2023-05-23T16:05:02.571713Z","shell.execute_reply":"2023-05-23T16:05:02.580868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define a DataFrame for the estimation results.","metadata":{}},{"cell_type":"code","source":"results_df = pd.DataFrame(columns=['model', 'accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:02.584270Z","iopub.execute_input":"2023-05-23T16:05:02.584541Z","iopub.status.idle":"2023-05-23T16:05:02.594061Z","shell.execute_reply.started":"2023-05-23T16:05:02.584512Z","shell.execute_reply":"2023-05-23T16:05:02.593152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Multilayer Perceptron ","metadata":{}},{"cell_type":"markdown","source":"First let's create and check the performance of MLP. Best params (hidden layers, neurons, lr etc) we will search with BayesianOptimization Keras tuner.","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nclass MLPHyperModel(kt.HyperModel):\n def build(self, hp):\n n_hidden = hp.Int(\"n_hidden\", min_value=1, max_value=4, default=2)\n n_neurons = hp.Int(\"n_neurons\", min_value=75, max_value=500, step=50)\n dropout_rate = hp.Float(\"dropout_rate\", min_value=0.1, max_value=0.8)\n learning_rate = hp.Float(\"learning_rate\", min_value=1e-4, max_value=1e-3,\n sampling=\"log\")\n optimizer = hp.Choice(\"optimizer\", values=[\"sgd\", \"adam\", \"nadam\", \"rmsprop\"])\n if optimizer == \"sgd\":\n optimizer = SGD(learning_rate=learning_rate)\n if optimizer == \"adam\":\n optimizer = Adam(learning_rate=learning_rate)\n if optimizer == \"nadam\":\n optimizer = Nadam(learning_rate=learning_rate)\n if optimizer == \"rmsprop\":\n optimizer = RMSprop(learning_rate=learning_rate)\n model = tf.keras.Sequential()\n model.add(Input((28,28,1)))\n model.add(Flatten())\n for _ in range(n_hidden):\n model.add(Dense(n_neurons, activation=\"relu\"))\n model.add(Dropout(dropout_rate))\n model.add(Dense(10, activation=\"softmax\"))\n model.compile(loss=tf.keras.losses.categorical_crossentropy, \n optimizer=optimizer,\n metrics=[tf.keras.metrics.categorical_accuracy]\n )\n return model\n\n def fit(self, hp, model, *args, **kwargs):\n batch_size = hp.Choice(name=\"batch_size\", values=[32, 64, 128, 256, 512])\n return model.fit(\n *args,\n batch_size=batch_size,\n **kwargs,\n )","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:02.595296Z","iopub.execute_input":"2023-05-23T16:05:02.596236Z","iopub.status.idle":"2023-05-23T16:05:02.619213Z","shell.execute_reply.started":"2023-05-23T16:05:02.596206Z","shell.execute_reply":"2023-05-23T16:05:02.618394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bayesian_opt_tuner = kt.BayesianOptimization(\n MLPHyperModel(), \n objective=\"val_categorical_accuracy\", \n seed=42,\n max_trials=20, \n alpha=1e-4, \n beta=2.6, \n overwrite = True\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:02.620402Z","iopub.execute_input":"2023-05-23T16:05:02.620750Z","iopub.status.idle":"2023-05-23T16:05:04.983871Z","shell.execute_reply.started":"2023-05-23T16:05:02.620718Z","shell.execute_reply":"2023-05-23T16:05:04.982956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bayesian_opt_tuner.search(\n X_train, y_train, \n epochs=20, validation_split=0.15,\n callbacks=[EarlyStopping('val_loss', patience=3)])","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:05:04.992504Z","iopub.execute_input":"2023-05-23T16:05:04.992798Z","iopub.status.idle":"2023-05-23T16:13:10.709385Z","shell.execute_reply.started":"2023-05-23T16:05:04.992774Z","shell.execute_reply":"2023-05-23T16:13:10.708341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_report(bayesian_opt_tuner)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:13:10.711986Z","iopub.execute_input":"2023-05-23T16:13:10.712637Z","iopub.status.idle":"2023-05-23T16:13:10.758497Z","shell.execute_reply.started":"2023-05-23T16:13:10.712603Z","shell.execute_reply":"2023-05-23T16:13:10.757564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = bayesian_opt_tuner.get_best_hyperparameters(num_trials=1)[0].values\nbest_params","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:13:10.759827Z","iopub.execute_input":"2023-05-23T16:13:10.760143Z","iopub.status.idle":"2023-05-23T16:13:10.766947Z","shell.execute_reply.started":"2023-05-23T16:13:10.760112Z","shell.execute_reply":"2023-05-23T16:13:10.765913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_mlp = bayesian_opt_tuner.get_best_models(num_models=1)[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:13:10.768737Z","iopub.execute_input":"2023-05-23T16:13:10.769513Z","iopub.status.idle":"2023-05-23T16:13:11.172407Z","shell.execute_reply.started":"2023-05-23T16:13:10.769479Z","shell.execute_reply":"2023-05-23T16:13:11.171493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_mlp.summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:13:11.173873Z","iopub.execute_input":"2023-05-23T16:13:11.174226Z","iopub.status.idle":"2023-05-23T16:13:11.200092Z","shell.execute_reply.started":"2023-05-23T16:13:11.174191Z","shell.execute_reply":"2023-05-23T16:13:11.199385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_mlp_history = best_mlp.fit(\n img_gen.flow(X_train, y_train),\n epochs=50,\n batch_size=best_params['batch_size'],\n validation_data=(X_val, y_val),\n verbose=1,\n callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T16:13:11.201065Z","iopub.execute_input":"2023-05-23T16:13:11.201373Z","iopub.status.idle":"2023-05-23T16:20:13.244559Z","shell.execute_reply.started":"2023-05-23T16:13:11.201343Z","shell.execute_reply":"2023-05-23T16:20:13.243636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(best_mlp, X_val, y_val.argmax(axis=-1), [], [], \"MLP\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:20:13.246194Z","iopub.execute_input":"2023-05-23T16:20:13.246525Z","iopub.status.idle":"2023-05-23T16:20:16.174387Z","shell.execute_reply.started":"2023-05-23T16:20:13.246493Z","shell.execute_reply":"2023-05-23T16:20:16.173556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = best_mlp.evaluate(X_val, y_val)\nresults_df = add_to_results(\"MLP\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:20:39.931707Z","iopub.execute_input":"2023-05-23T18:20:39.932720Z","iopub.status.idle":"2023-05-23T18:20:40.674163Z","shell.execute_reply.started":"2023-05-23T18:20:39.932659Z","shell.execute_reply":"2023-05-23T18:20:40.673222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LeNet-5 ","metadata":{}},{"cell_type":"markdown","source":"LeNet-5 was created by Yann LeCun in 1998 and has been widely used for handwritten digit recognition (MNIST).\n\nThe famous LeNet-5 architecture had the following layers:\n\nLayer | Type | Maps | Size | Kernel size | Stride | Activation\n-------|-----------------|------|----------|-------------|--------|-----------\n Out | Fully connected | – | 10 | – | – | RBF\n F6 | Fully connected | – | 84 | – | – | tanh\n C5 | Convolution | 120 | 1 × 1 | 5 × 5 | 1 | tanh\n S4 | Avg pooling | 16 | 5 × 5 | 2 × 2 | 2 | tanh\n C3 | Convolution | 16 | 10 × 10 | 5 × 5 | 1 | tanh\n S2 | Avg pooling | 6 | 14 × 14 | 2 × 2 | 2 | tanh\n C1 | Convolution | 6 | 28 × 28 | 5 × 5 | 1 | tanh\n In | Input | 1 | 32 × 32 | – | – | –\n\n\nCreate this model using the Subclassing API (with a few tweaks like relu activation function instead of tanh and so on).","metadata":{}},{"cell_type":"code","source":"class LeNet(Sequential):\n def __init__(self, input_shape, nb_classes):\n super().__init__()\n self.add(Conv2D(6, kernel_size=(5, 5), strides=(1, 1), activation='relu', input_shape=input_shape, padding=\"same\"))\n self.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))\n self.add(Conv2D(16, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='valid'))\n self.add(AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))\n self.add(Flatten())\n self.add(Dense(120, activation='relu'))\n self.add(Dense(84, activation='relu'))\n self.add(Dense(nb_classes, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:21:08.820542Z","iopub.execute_input":"2023-05-23T16:21:08.821362Z","iopub.status.idle":"2023-05-23T16:21:08.830990Z","shell.execute_reply.started":"2023-05-23T16:21:08.821327Z","shell.execute_reply":"2023-05-23T16:21:08.830030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try to fine-tune the next hyperparams that are not related to the models' architecture: \n* optimizer\n* learning_rate (of Optimizer)\n* factor (from ReduceLROnPlateau)\n* batch_size\n\nTo do this, create the model_builder function, which creates the model from the LeNet class and defines the hyperparam search space for compiling the model, such as optimizer and learning rate.","metadata":{}},{"cell_type":"code","source":"def model_builder(hp):\n model = LeNet(X_train[0].shape, 10)\n learning_rate = hp.Choice('learning_rate', values=[0.001, 0.005, 0.01])\n optimizer = hp.Choice(\"optimizer\", values=[\"adam\", \"nadam\", \"rmsprop\"])\n if optimizer == \"adam\":\n optimizer = Adam(learning_rate=learning_rate)\n if optimizer == \"nadam\":\n optimizer = Nadam(learning_rate=learning_rate)\n if optimizer == \"rmsprop\":\n optimizer = RMSprop(learning_rate=learning_rate)\n model.compile(loss=categorical_crossentropy, \n optimizer=optimizer,\n metrics=[categorical_accuracy]\n )\n return model","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:21:11.499484Z","iopub.execute_input":"2023-05-23T16:21:11.499848Z","iopub.status.idle":"2023-05-23T16:21:11.506780Z","shell.execute_reply.started":"2023-05-23T16:21:11.499820Z","shell.execute_reply":"2023-05-23T16:21:11.505909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Learning rate scheduling is an important part of model learning. In this notebook I use ReduceLROnPlateau. The parameter factor of this class defines the rate by which the learning rate is reduced. The default value of 0.1 seems to be too low. One of the approaches to fine-tune the parameters contained within the model fit method (this is also the batch_size parameter) is to create a subclass of one of the kt.tuners classes and redefine the run_trial method.","metadata":{}},{"cell_type":"code","source":"class CustomBayesTuner(kt.tuners.BayesianOptimization):\n def run_trial(self, trial, *args, **kwargs):\n hp = trial.hyperparameters\n factor = hp.Float('factor', min_value=0.1, max_value=0.5, step=0.1)\n rlrop = ReduceLROnPlateau(monitor='val_categorical_accuracy', \n patience=3, verbose=1, factor=factor)\n es = EarlyStopping(\n monitor='val_categorical_accuracy', \n patience=7,\n verbose=1)\n kwargs['callbacks'] = [rlrop, es]\n kwargs['batch_size'] = hp.Choice(name=\"batch_size\", values=[32, 64, 128, 256, 512])\n return super(CustomBayesTuner, self).run_trial(trial, *args, **kwargs)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:21:13.701570Z","iopub.execute_input":"2023-05-23T16:21:13.702561Z","iopub.status.idle":"2023-05-23T16:21:13.710724Z","shell.execute_reply.started":"2023-05-23T16:21:13.702516Z","shell.execute_reply":"2023-05-23T16:21:13.709816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bayesian_opt_tuner = CustomBayesTuner(\n model_builder, \n objective=\"val_categorical_accuracy\", \n seed=42,\n max_trials=20,\n overwrite = True\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:21:15.710113Z","iopub.execute_input":"2023-05-23T16:21:15.710516Z","iopub.status.idle":"2023-05-23T16:21:15.813388Z","shell.execute_reply.started":"2023-05-23T16:21:15.710486Z","shell.execute_reply":"2023-05-23T16:21:15.812489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nbayesian_opt_tuner.search(\n X_train, y_train, \n epochs=25,\n validation_split=0.4\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:21:17.810707Z","iopub.execute_input":"2023-05-23T16:21:17.811430Z","iopub.status.idle":"2023-05-23T16:46:03.451546Z","shell.execute_reply.started":"2023-05-23T16:21:17.811397Z","shell.execute_reply":"2023-05-23T16:46:03.450558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_report(bayesian_opt_tuner)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:46:03.453554Z","iopub.execute_input":"2023-05-23T16:46:03.454014Z","iopub.status.idle":"2023-05-23T16:46:03.498504Z","shell.execute_reply.started":"2023-05-23T16:46:03.453980Z","shell.execute_reply":"2023-05-23T16:46:03.497504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = bayesian_opt_tuner.get_best_hyperparameters(num_trials=1)[0].values\nbest_params","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:46:03.499832Z","iopub.execute_input":"2023-05-23T16:46:03.500233Z","iopub.status.idle":"2023-05-23T16:46:03.507995Z","shell.execute_reply.started":"2023-05-23T16:46:03.500200Z","shell.execute_reply":"2023-05-23T16:46:03.506964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_rlrop_factor = best_params['factor']\nbest_rlrop = ReduceLROnPlateau(monitor='val_categorical_accuracy', \n patience=3, verbose=1, factor=best_rlrop_factor)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:46:03.510664Z","iopub.execute_input":"2023-05-23T16:46:03.511430Z","iopub.status.idle":"2023-05-23T16:46:03.516466Z","shell.execute_reply.started":"2023-05-23T16:46:03.511381Z","shell.execute_reply":"2023-05-23T16:46:03.515488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntf.keras.backend.clear_session()\n\nmodel_lenet = LeNet(X_train[0].shape, 10)\nmodel_lenet.compile(loss=categorical_crossentropy, \n optimizer=Adam(best_params['learning_rate']), \n metrics=[categorical_accuracy]\n )\nmodel_lenet_history = model_lenet.fit(\n img_gen.flow(X_train, y_train),\n epochs=50,\n batch_size=best_params['batch_size'],\n validation_data=(X_val, y_val),\n verbose=1,\n callbacks=[early_stopping, best_rlrop]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T16:46:03.517880Z","iopub.execute_input":"2023-05-23T16:46:03.518234Z","iopub.status.idle":"2023-05-23T16:54:15.195072Z","shell.execute_reply.started":"2023-05-23T16:46:03.518201Z","shell.execute_reply":"2023-05-23T16:54:15.194037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(model_lenet, X_val, y_val.argmax(axis=-1), [], [], \"LeNet-5\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:54:15.197533Z","iopub.execute_input":"2023-05-23T16:54:15.198566Z","iopub.status.idle":"2023-05-23T16:54:17.948437Z","shell.execute_reply.started":"2023-05-23T16:54:15.198532Z","shell.execute_reply":"2023-05-23T16:54:17.947562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thus, we achieved good performance with a relatively simple CNN using only 2 Conv2D layers. Now let's try deeper CNNs with the same model training hyperparams.","metadata":{}},{"cell_type":"code","source":"_, acc = model_lenet.evaluate(X_val, y_val)\nresults_df = add_to_results(\"LeNet-5\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:20:49.337892Z","iopub.execute_input":"2023-05-23T18:20:49.338250Z","iopub.status.idle":"2023-05-23T18:20:50.083100Z","shell.execute_reply.started":"2023-05-23T18:20:49.338223Z","shell.execute_reply":"2023-05-23T18:20:50.081967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## AlexNet ","metadata":{}},{"cell_type":"markdown","source":"The AlexNet CNN architecture won the 2012 ILSVRC challenge. Let's check the basic architecture of this CNN with the MNIST classification task.\n\nBecause of the smaller input size (28x28; and better results) I use here some changes in the architecture, like kernel size of the first Conv2D layers, only 1 last dense layer with less neurons. ","metadata":{}},{"cell_type":"code","source":"model_alexnet = tf.keras.models.Sequential([\n Conv2D(filters=96, kernel_size=(5,5), strides=(1,1), activation='relu', padding='same', input_shape=(28,28,1)),\n BatchNormalization(),\n MaxPool2D(pool_size=(3,3), strides=(2,2)),\n Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n BatchNormalization(),\n MaxPool2D(pool_size=(2,2), strides=(2,2)),\n Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n BatchNormalization(),\n Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n BatchNormalization(),\n Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\"),\n BatchNormalization(),\n MaxPool2D(pool_size=(3,3), strides=(2,2)),\n Flatten(),\n# Dense(4096, activation='relu'),\n Dense(1024, activation='relu'),\n Dropout(0.4), # change to 0.2\n# Dense(4096, activation='relu'),\n# Dropout(0.4),\n Dense(10, activation='softmax')\n])\nmodel_alexnet.compile(loss=categorical_crossentropy, \n optimizer=Adam(0.005), \n# optimizer=RMSprop(),\n metrics=[categorical_accuracy]\n )","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:55:00.286453Z","iopub.execute_input":"2023-05-23T16:55:00.286849Z","iopub.status.idle":"2023-05-23T16:55:00.475127Z","shell.execute_reply.started":"2023-05-23T16:55:00.286819Z","shell.execute_reply":"2023-05-23T16:55:00.474262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_alexnet.summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T16:55:04.242036Z","iopub.execute_input":"2023-05-23T16:55:04.242397Z","iopub.status.idle":"2023-05-23T16:55:04.290970Z","shell.execute_reply.started":"2023-05-23T16:55:04.242370Z","shell.execute_reply":"2023-05-23T16:55:04.290170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntf.keras.backend.clear_session()\n\nmodel_alexnet_history = model_alexnet.fit(\n img_gen.flow(X_train, y_train),\n batch_size=64,\n epochs=50,\n validation_data=(X_val, y_val),\n verbose=1,\n callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T16:55:08.255190Z","iopub.execute_input":"2023-05-23T16:55:08.255541Z","iopub.status.idle":"2023-05-23T17:06:37.388525Z","shell.execute_reply.started":"2023-05-23T16:55:08.255513Z","shell.execute_reply":"2023-05-23T17:06:37.387522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(model_alexnet, X_val, y_val.argmax(axis=-1), [], [], \"AlexNet\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:06:37.390750Z","iopub.execute_input":"2023-05-23T17:06:37.391152Z","iopub.status.idle":"2023-05-23T17:06:44.854380Z","shell.execute_reply.started":"2023-05-23T17:06:37.391118Z","shell.execute_reply":"2023-05-23T17:06:44.853554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = model_alexnet.evaluate(X_val, y_val)\nresults_df = add_to_results(\"AlexNet\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:21:00.269754Z","iopub.execute_input":"2023-05-23T18:21:00.270117Z","iopub.status.idle":"2023-05-23T18:21:01.681346Z","shell.execute_reply.started":"2023-05-23T18:21:00.270090Z","shell.execute_reply":"2023-05-23T18:21:01.680517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Errors visualization","metadata":{}},{"cell_type":"markdown","source":"Let's look at some errors.","metadata":{}},{"cell_type":"code","source":"plot_errors(model_alexnet, X_val, y_val.argmax(axis=-1), 28, 7)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:28.910955Z","iopub.execute_input":"2023-05-23T17:25:28.911306Z","iopub.status.idle":"2023-05-23T17:25:33.050423Z","shell.execute_reply.started":"2023-05-23T17:25:28.911279Z","shell.execute_reply":"2023-05-23T17:25:33.049442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that some digits are difficult to recognize even for humans.","metadata":{}},{"cell_type":"markdown","source":"## ResNet ","metadata":{}},{"cell_type":"markdown","source":"Residual Network (ResNet) won ILSVRC 2015 challenge. The winning variant used an extremely deep CNN composed of 152 layers (other variants had 18, 34, 50, and 101 layers). The key to this architecture is the use of skip connections (also called shortcut connections).","metadata":{}},{"cell_type":"markdown","source":"First, create a ResidualUnit class with a shortcut connection.","metadata":{}},{"cell_type":"code","source":"DefaultConv2D = partial(tf.keras.layers.Conv2D, kernel_size=3, strides=1,\n padding=\"same\", kernel_initializer=\"he_normal\",\n use_bias=False)\n\nclass ResidualUnit(tf.keras.layers.Layer):\n def __init__(self, filters, strides=1, activation=\"relu\", **kwargs):\n super().__init__(**kwargs)\n self.activation = tf.keras.activations.get(activation)\n self.main_layers = [\n DefaultConv2D(filters, strides=strides),\n tf.keras.layers.BatchNormalization(),\n self.activation,\n DefaultConv2D(filters),\n tf.keras.layers.BatchNormalization()\n ]\n self.skip_layers = []\n if strides > 1:\n self.skip_layers = [\n DefaultConv2D(filters, kernel_size=1, strides=strides),\n tf.keras.layers.BatchNormalization()\n ]\n\n def call(self, inputs):\n x = inputs\n for layer in self.main_layers:\n x = layer(x)\n skip_x = inputs\n for layer in self.skip_layers:\n skip_x = layer(skip_x)\n return self.activation(x + skip_x)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:38.355894Z","iopub.execute_input":"2023-05-23T17:25:38.356260Z","iopub.status.idle":"2023-05-23T17:25:38.366241Z","shell.execute_reply.started":"2023-05-23T17:25:38.356232Z","shell.execute_reply":"2023-05-23T17:25:38.364653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I will also test a pre-activation ResidualUnit (from Pre-ResNet). It differs from the ordinal ResidualUnit by the position of the BatchNormalization and Activation layers before the Conv2D layer.","metadata":{}},{"cell_type":"code","source":"class PreResidualUnit(tf.keras.layers.Layer):\n def __init__(self, filters, strides=1, activation=\"relu\", **kwargs): #drop_rate=0.1, \n super().__init__(**kwargs)\n self.activation = tf.keras.activations.get(activation)\n self.main_layers = [\n tf.keras.layers.BatchNormalization(),\n self.activation,\n DefaultConv2D(filters, strides=strides),\n tf.keras.layers.BatchNormalization(),\n self.activation,\n DefaultConv2D(filters),\n ]\n self.skip_layers = []\n if strides > 1:\n self.skip_layers = [\n DefaultConv2D(filters, kernel_size=1, strides=strides),\n tf.keras.layers.BatchNormalization()\n ]\n\n def call(self, inputs):\n x = inputs\n for layer in self.main_layers:\n x = layer(x)\n skip_x = inputs\n for layer in self.skip_layers:\n skip_x = layer(skip_x)\n return self.activation(x + skip_x)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:40.159577Z","iopub.execute_input":"2023-05-23T17:25:40.160063Z","iopub.status.idle":"2023-05-23T17:25:40.170718Z","shell.execute_reply.started":"2023-05-23T17:25:40.160029Z","shell.execute_reply":"2023-05-23T17:25:40.169736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a custom ResNet model using the Subclassing API and have just created a ResidualUnit class.","metadata":{}},{"cell_type":"markdown","source":"For experimentation, I created an initial layer (which is also an optional variable for the CustomResNet class below) consisting of two Conv2D layers with different kernel shapes. Input is passed separately to these Conv2D layers, which are concatenated, and then processed by subsequent parts of the network.","metadata":{}},{"cell_type":"code","source":"class ConcatLayer(tf.keras.layers.Layer):\n def __init__(self, filters=32, activation=\"relu\", kernel_1 = (8,3), kernel_2 = (3,8), \n inp_shape=(28, 28, 1), **kwargs):\n super().__init__(**kwargs)\n self.conv_1 = Conv2D(filters = filters, kernel_size = kernel_1, padding = 'same', activation = activation, input_shape=inp_shape)\n self.conv_2 = Conv2D(filters = filters, kernel_size = kernel_2, padding = 'same', activation = activation, input_shape=inp_shape)\n\n def call(self, inputs):\n x = inputs\n output = concatenate([self.conv_1(x), self.conv_2(x)], axis = -1)\n return output","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:42.384497Z","iopub.execute_input":"2023-05-23T17:25:42.384962Z","iopub.status.idle":"2023-05-23T17:25:42.397634Z","shell.execute_reply.started":"2023-05-23T17:25:42.384921Z","shell.execute_reply":"2023-05-23T17:25:42.396847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomResNet(tf.keras.Model):\n def __init__(self, residual_filters, init_kernel=(5,5), inp_shape=(28, 28, 1), num_classes=10,\n drop_rate=0.4, last_units=512, preres=False, concat_layer=False, **kwargs):\n super().__init__(**kwargs) \n self.drop_rate = drop_rate\n self.inp_shape = inp_shape\n self.preres = preres\n if concat_layer:\n concat_filters = residual_filters[0] / 2\n init_layer = ConcatLayer(filters=concat_filters)\n residual_filters = np.array(residual_filters)\n else:\n init_layer = Conv2D(residual_filters[0], kernel_size=init_kernel,\n strides=1, input_shape=inp_shape,\n padding=\"same\", kernel_initializer=\"he_normal\",\n use_bias=False)\n self.first_layers = [\n init_layer,\n BatchNormalization(),\n Activation(\"relu\"),\n MaxPool2D(pool_size=2, strides=2, padding=\"valid\"),\n ]\n self.residual_layers = self._get_residual_layers(residual_filters)\n self.last_layers = [\n GlobalAvgPool2D(),\n Flatten(),\n Dropout(drop_rate),\n Dense(last_units, activation=\"relu\", \n kernel_initializer=\"he_normal\"),\n ]\n self.activation = Dense(num_classes, activation=\"softmax\")\n \n def _get_residual_layers(self, filters):\n residual_layers = []\n prev_f = filters[0]\n for f in filters:\n strides = 1 if f == prev_f else 2\n resunit = PreResidualUnit(f, strides=strides) if self.preres else ResidualUnit(f, strides=strides)\n residual_layers.append(resunit)\n prev_f = f\n return residual_layers\n \n def call(self, inputs, training=False):\n x = inputs\n for layer in self.first_layers + self.residual_layers + self.last_layers:\n x = layer(x)\n output = self.activation(x)\n return output\n \n def build_graph(self):\n x = Input(shape=self.inp_shape)\n return Model(inputs=[x], outputs=self.call(x))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:44.137203Z","iopub.execute_input":"2023-05-23T17:25:44.137568Z","iopub.status.idle":"2023-05-23T17:25:44.153595Z","shell.execute_reply.started":"2023-05-23T17:25:44.137537Z","shell.execute_reply":"2023-05-23T17:25:44.152650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I am creating 3 different custom variants of ResNet.","metadata":{}},{"cell_type":"markdown","source":"### Custom ResNet (pre)","metadata":{}},{"cell_type":"code","source":"res_filters1 = [64] * 2 + [128] * 2 + [256] * 2 + [512] * 1\nmodel_customresnet1 = CustomResNet(res_filters1, preres=True, concat_layer=False)\nmodel_customresnet1.compile(loss=categorical_crossentropy, \n optimizer=Adam(0.005), \n metrics=[categorical_accuracy]\n )","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:46.548603Z","iopub.execute_input":"2023-05-23T17:25:46.549106Z","iopub.status.idle":"2023-05-23T17:25:46.650997Z","shell.execute_reply.started":"2023-05-23T17:25:46.549075Z","shell.execute_reply":"2023-05-23T17:25:46.650011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_customresnet1.build_graph().summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:25:51.033021Z","iopub.execute_input":"2023-05-23T17:25:51.033372Z","iopub.status.idle":"2023-05-23T17:25:51.471265Z","shell.execute_reply.started":"2023-05-23T17:25:51.033345Z","shell.execute_reply":"2023-05-23T17:25:51.470525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntf.keras.backend.clear_session()\n\nmodel_customresnet1_history = model_customresnet1.fit(\n img_gen.flow(X_train, y_train),\n batch_size=64,\n epochs=50,\n validation_data=(X_val, y_val),\n verbose=1,\n callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T17:25:57.223342Z","iopub.execute_input":"2023-05-23T17:25:57.223734Z","iopub.status.idle":"2023-05-23T17:35:15.968583Z","shell.execute_reply.started":"2023-05-23T17:25:57.223700Z","shell.execute_reply":"2023-05-23T17:35:15.967523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(model_customresnet1, X_val, y_val.argmax(axis=-1), [], [], \"Custom ResNet (pre)\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:35:15.970842Z","iopub.execute_input":"2023-05-23T17:35:15.971207Z","iopub.status.idle":"2023-05-23T17:35:20.474576Z","shell.execute_reply.started":"2023-05-23T17:35:15.971173Z","shell.execute_reply":"2023-05-23T17:35:20.473632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = model_customresnet1.evaluate(X_val, y_val)\nresults_df = add_to_results(\"ResNet (pre)\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:21:12.346520Z","iopub.execute_input":"2023-05-23T18:21:12.347504Z","iopub.status.idle":"2023-05-23T18:21:13.810012Z","shell.execute_reply.started":"2023-05-23T18:21:12.347468Z","shell.execute_reply":"2023-05-23T18:21:13.809158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Custom ResNet (concat layer)","metadata":{}},{"cell_type":"code","source":"res_filters1 = [64] * 2 + [128] * 2 + [256] * 2 + [512] * 1\nmodel_customresnet2 = CustomResNet(res_filters1, preres=False, concat_layer=True)\nmodel_customresnet2.compile(loss=categorical_crossentropy, \n optimizer=Adam(0.005), \n metrics=[categorical_accuracy]\n )","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:35:21.479002Z","iopub.execute_input":"2023-05-23T17:35:21.479293Z","iopub.status.idle":"2023-05-23T17:35:21.586124Z","shell.execute_reply.started":"2023-05-23T17:35:21.479268Z","shell.execute_reply":"2023-05-23T17:35:21.585070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_customresnet2.build_graph().summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:35:21.590602Z","iopub.execute_input":"2023-05-23T17:35:21.590981Z","iopub.status.idle":"2023-05-23T17:35:22.207379Z","shell.execute_reply.started":"2023-05-23T17:35:21.590950Z","shell.execute_reply":"2023-05-23T17:35:22.206702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model = model_customresnet2.build_graph(), dpi=72, show_shapes=True, expand_nested=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T17:35:22.208416Z","iopub.execute_input":"2023-05-23T17:35:22.218910Z","iopub.status.idle":"2023-05-23T17:35:22.714764Z","shell.execute_reply.started":"2023-05-23T17:35:22.218875Z","shell.execute_reply":"2023-05-23T17:35:22.712904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntf.keras.backend.clear_session()\n\nmodel_customresnet2_history = model_customresnet2.fit(\n img_gen.flow(X_train, y_train),\n batch_size=64, #64\n epochs=50,\n validation_data=(X_val, y_val),\n verbose=1,\n callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T17:35:22.717724Z","iopub.execute_input":"2023-05-23T17:35:22.720883Z","iopub.status.idle":"2023-05-23T17:45:26.547373Z","shell.execute_reply.started":"2023-05-23T17:35:22.720849Z","shell.execute_reply":"2023-05-23T17:45:26.546399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(model_customresnet2, X_val, y_val.argmax(axis=-1), [], [], \"Custom ResNet (concat)\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:45:26.548707Z","iopub.execute_input":"2023-05-23T17:45:26.549634Z","iopub.status.idle":"2023-05-23T17:45:30.642219Z","shell.execute_reply.started":"2023-05-23T17:45:26.549601Z","shell.execute_reply":"2023-05-23T17:45:30.641360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = model_customresnet2.evaluate(X_val, y_val)\nresults_df = add_to_results(\"ResNet (concat)\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:21:18.853105Z","iopub.execute_input":"2023-05-23T18:21:18.853460Z","iopub.status.idle":"2023-05-23T18:21:20.315571Z","shell.execute_reply.started":"2023-05-23T18:21:18.853434Z","shell.execute_reply":"2023-05-23T18:21:20.314728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Custom ResNet (ordinal)","metadata":{}},{"cell_type":"code","source":"res_filters1 = [64] * 2 + [128] * 2 + [256] * 2 + [512] * 1\nmodel_customresnet3 = CustomResNet(res_filters1, preres=False, concat_layer=False)\nmodel_customresnet3.compile(loss=categorical_crossentropy, \n optimizer=Adam(0.005), \n metrics=[categorical_accuracy]\n )","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:45:31.556558Z","iopub.execute_input":"2023-05-23T17:45:31.556845Z","iopub.status.idle":"2023-05-23T17:45:31.624847Z","shell.execute_reply.started":"2023-05-23T17:45:31.556821Z","shell.execute_reply":"2023-05-23T17:45:31.624037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_customresnet3.build_graph().summary()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:45:31.626769Z","iopub.execute_input":"2023-05-23T17:45:31.627334Z","iopub.status.idle":"2023-05-23T17:45:31.959528Z","shell.execute_reply.started":"2023-05-23T17:45:31.627304Z","shell.execute_reply":"2023-05-23T17:45:31.958748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel_customresnet3_history = model_customresnet3.fit(\n img_gen.flow(X_train, y_train),\n batch_size=64, #64\n epochs=50,\n validation_data=(X_val, y_val),\n verbose=1,\n callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T17:45:31.960801Z","iopub.execute_input":"2023-05-23T17:45:31.961134Z","iopub.status.idle":"2023-05-23T17:57:49.003517Z","shell.execute_reply.started":"2023-05-23T17:45:31.961101Z","shell.execute_reply":"2023-05-23T17:57:49.002393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(model_customresnet3, X_val, y_val.argmax(axis=-1), [], [], \"Custom ResNet (ordinal)\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T17:57:49.004886Z","iopub.execute_input":"2023-05-23T17:57:49.005512Z","iopub.status.idle":"2023-05-23T17:57:52.580533Z","shell.execute_reply.started":"2023-05-23T17:57:49.005478Z","shell.execute_reply":"2023-05-23T17:57:52.579640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = model_customresnet3.evaluate(X_val, y_val)\nresults_df = add_to_results(\"ResNet (ordinal)\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:21:28.044470Z","iopub.execute_input":"2023-05-23T18:21:28.044862Z","iopub.status.idle":"2023-05-23T18:21:29.504275Z","shell.execute_reply.started":"2023-05-23T18:21:28.044834Z","shell.execute_reply":"2023-05-23T18:21:29.503371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MobileNet (Transfer Learning) ","metadata":{}},{"cell_type":"markdown","source":"Let's evaluate one of the models from Keras Applications for the Mnist classification task. In particular, I chose MobileNet because it's not so deep and relatively fast. First, using the Keras GridSearch tuner, I evaluate how it performs without weights, with locked weights (except of the last dense layers), and with unlocked weights.","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.clear_session()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:32:18.695493Z","iopub.execute_input":"2023-05-23T18:32:18.695894Z","iopub.status.idle":"2023-05-23T18:32:18.809732Z","shell.execute_reply.started":"2023-05-23T18:32:18.695864Z","shell.execute_reply":"2023-05-23T18:32:18.808796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_getter(app_class):\n def get_model(include_top=False, weights=None, layouts_locked=False, learning_rate=0.0005):\n model = app_class(\n input_shape=(32,32,3),\n include_top=include_top,\n weights=weights,\n )\n x = model.output\n if not include_top:\n x = Flatten(name=f\"flatten_{str(time.time())[:-14]}\")(x)\n x = Dense(128, activation=\"relu\", kernel_initializer=\"he_normal\")(x)\n x = Dropout(0.2)(x)\n output = Dense(10, activation=\"softmax\")(x)\n model = Model(inputs=model.input, outputs=output)\n for layer in model.layers[:-3]:\n layer.trainable = not layouts_locked\n optimizer = Adam(learning_rate=learning_rate)\n model.compile(loss=categorical_crossentropy, \n optimizer=optimizer,\n metrics=[categorical_accuracy]\n )\n return model\n return get_model","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:45:46.539128Z","iopub.execute_input":"2023-05-23T19:45:46.539491Z","iopub.status.idle":"2023-05-23T19:45:46.550203Z","shell.execute_reply.started":"2023-05-23T19:45:46.539461Z","shell.execute_reply":"2023-05-23T19:45:46.549238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_app_builder(hp):\n model_type = hp.Choice('model_type', values=[\"no_weights\", \"weights_locked\", \"weights_unlocked\"])\n model_getter_fx = model_getter(tf.keras.applications.MobileNet)\n with hp.conditional_scope(\"model_type\", [\"no_weights\"]):\n if model_type == \"no_weights\":\n model_include_top = False\n model_weights = None\n learning_rate = 0.005\n model = model_getter_fx(model_include_top, model_weights, False, learning_rate)\n with hp.conditional_scope(\"model_type\", [\"weights_locked\", \"weights_unlocked\"]):\n if model_type in [\"weights_locked\", \"weights_unlocked\"]:\n model_include_top = False\n model_weights = \"imagenet\"\n learning_rate = 0.0005\n layouts_locked = True if model_type == \"weights_locked\" else False\n model = model_getter_fx(model_include_top, model_weights, layouts_locked, learning_rate)\n return model","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:42:58.627709Z","iopub.execute_input":"2023-05-23T18:42:58.628619Z","iopub.status.idle":"2023-05-23T18:42:58.636723Z","shell.execute_reply.started":"2023-05-23T18:42:58.628587Z","shell.execute_reply":"2023-05-23T18:42:58.635745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_tuner = kt.GridSearch(\n hypermodel=model_app_builder,\n objective=\"val_categorical_accuracy\",\n seed=42,\n overwrite=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:43:00.846121Z","iopub.execute_input":"2023-05-23T18:43:00.846480Z","iopub.status.idle":"2023-05-23T18:43:02.099474Z","shell.execute_reply.started":"2023-05-23T18:43:00.846452Z","shell.execute_reply":"2023-05-23T18:43:02.098321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_tuner.search(\n X_train_32_3, y_train,\n batch_size=64,\n epochs=30,\n validation_split=0.4,\n callbacks=[ReduceLROnPlateau(monitor='val_categorical_accuracy',\n patience=3,\n verbose=1, \n factor=0.3)]\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:43:04.164043Z","iopub.execute_input":"2023-05-23T18:43:04.164721Z","iopub.status.idle":"2023-05-23T18:55:30.258560Z","shell.execute_reply.started":"2023-05-23T18:43:04.164661Z","shell.execute_reply":"2023-05-23T18:55:30.257570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_report(grid_search_tuner)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:55:35.096772Z","iopub.execute_input":"2023-05-23T18:55:35.097124Z","iopub.status.idle":"2023-05-23T18:55:35.110250Z","shell.execute_reply.started":"2023-05-23T18:55:35.097098Z","shell.execute_reply":"2023-05-23T18:55:35.109349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_mobilenet = model_getter(tf.keras.applications.MobileNet)(include_top=False, weights='imagenet', \n layouts_locked=False, learning_rate=0.001)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:45:53.143721Z","iopub.execute_input":"2023-05-23T19:45:53.144511Z","iopub.status.idle":"2023-05-23T19:45:53.859825Z","shell.execute_reply.started":"2023-05-23T19:45:53.144479Z","shell.execute_reply":"2023-05-23T19:45:53.858865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_mobilenet.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-23T19:46:03.410542Z","iopub.execute_input":"2023-05-23T19:46:03.411116Z","iopub.status.idle":"2023-05-23T19:46:03.556893Z","shell.execute_reply.started":"2023-05-23T19:46:03.411083Z","shell.execute_reply":"2023-05-23T19:46:03.556195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntf.keras.backend.clear_session()\n\nmodel_mobilenet_history = model_mobilenet.fit(\n img_gen.flow(X_train_32_3, y_train),\n batch_size=64,\n epochs=50,\n validation_data=(X_val_32_3, y_val),\n verbose=1,\n callbacks=[early_stopping, reduce_lr_on_plateau]\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:46:13.463492Z","iopub.execute_input":"2023-05-23T19:46:13.464322Z","iopub.status.idle":"2023-05-23T20:02:26.762545Z","shell.execute_reply.started":"2023-05-23T19:46:13.464290Z","shell.execute_reply":"2023-05-23T20:02:26.761543Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EstimatorReport(model_mobilenet, X_val_32_3, y_val.argmax(axis=-1), [], [], \"MobileNet\", True, 10).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:02:35.228285Z","iopub.execute_input":"2023-05-23T20:02:35.229323Z","iopub.status.idle":"2023-05-23T20:02:44.905831Z","shell.execute_reply.started":"2023-05-23T20:02:35.229285Z","shell.execute_reply":"2023-05-23T20:02:44.904696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = model_mobilenet.evaluate(X_val_32_3, y_val)\nresults_df = add_to_results(\"MobileNet\", acc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T19:25:57.437770Z","iopub.execute_input":"2023-05-23T19:25:57.438723Z","iopub.status.idle":"2023-05-23T19:25:58.753791Z","shell.execute_reply.started":"2023-05-23T19:25:57.438667Z","shell.execute_reply":"2023-05-23T19:25:58.752933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensembling ","metadata":{}},{"cell_type":"markdown","source":"So we have multiple trained models (see validation results below). Create a stacked ensemble.","metadata":{}},{"cell_type":"code","source":"display(results_df\n .sort_values(by='accuracy', ascending=False)\n .reset_index(drop=True)\n .style\n .background_gradient(cmap='viridis')\n .set_properties(**{'text-align': 'left'})\n )","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:05:00.468172Z","iopub.execute_input":"2023-05-23T20:05:00.468613Z","iopub.status.idle":"2023-05-23T20:05:00.491348Z","shell.execute_reply.started":"2023-05-23T20:05:00.468578Z","shell.execute_reply":"2023-05-23T20:05:00.490412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stacking_ensemble(models_28, models_32, X_28_1, X_32_3):\n stacked_predictions = []\n for model in models_28:\n predictions = model.predict(X_28_1)\n stacked_predictions.append(predictions)\n for model in models_32:\n predictions = model.predict(X_32_3)\n stacked_predictions.append(predictions)\n stacked_predictions = np.stack(stacked_predictions)\n final_predictions = stacked_predictions.mean(axis=0).argmax(axis=1)\n return final_predictions","metadata":{"execution":{"iopub.status.busy":"2023-05-23T18:21:56.733255Z","iopub.execute_input":"2023-05-23T18:21:56.733615Z","iopub.status.idle":"2023-05-23T18:21:56.742343Z","shell.execute_reply.started":"2023-05-23T18:21:56.733585Z","shell.execute_reply":"2023-05-23T18:21:56.741431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_for_voting = [model_lenet, model_alexnet, model_customresnet1, model_customresnet2, model_customresnet3]\nmodels_for_voting_32 = [model_mobilenet]\ny_test_pred = stacking_ensemble(models_for_voting, models_for_voting_32, X_test, X_test_32_3)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:06:57.945035Z","iopub.execute_input":"2023-05-23T20:06:57.945515Z","iopub.status.idle":"2023-05-23T20:07:17.772485Z","shell.execute_reply.started":"2023-05-23T20:06:57.945479Z","shell.execute_reply":"2023-05-23T20:07:17.771581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission ","metadata":{}},{"cell_type":"code","source":"sumbission = pd.DataFrame(\n {\n \"ImageId\": np.arange(1, len(X_test)+1),\n \"Label\": y_test_pred,\n }\n)\nsumbission.head()\nsumbission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T20:07:20.530986Z","iopub.execute_input":"2023-05-23T20:07:20.531646Z","iopub.status.idle":"2023-05-23T20:07:20.589299Z","shell.execute_reply.started":"2023-05-23T20:07:20.531603Z","shell.execute_reply":"2023-05-23T20:07:20.588485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion ","metadata":{}},{"cell_type":"markdown","source":"By training multiple CNN models using fine-tuning and ensembling, we achieved good performance (at least better than with classic Sklearn models).","metadata":{}}]}