{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-07-17T12:29:55.525759Z","iopub.execute_input":"2025-07-17T12:29:55.526269Z","iopub.status.idle":"2025-07-17T12:29:55.808322Z","shell.execute_reply.started":"2025-07-17T12:29:55.526243Z","shell.execute_reply":"2025-07-17T12:29:55.807537Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Credits\n\n##### This notebook has been taken from kaggle user \"jedrzejdudzicz\". Link: https://www.kaggle.com/jedrzejdudzicz\n\n##### Target Learning has been Implemented by me which actually increased the accuracy!\n\n##### Let me know what you guys think?","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nnp.random.seed(0) \nimport random\n\nimport tensorflow.keras as keras\nfrom tensorflow.keras.models import Sequential, load_model\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten\nfrom tensorflow.keras.layers import Conv2D, MaxPool2D, BatchNormalization\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n\nfrom tensorflow.keras.utils import plot_model\n\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:32:35.447187Z","iopub.execute_input":"2025-07-17T13:32:35.447795Z","iopub.status.idle":"2025-07-17T13:32:35.455143Z","shell.execute_reply.started":"2025-07-17T13:32:35.447771Z","shell.execute_reply":"2025-07-17T13:32:35.454436Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Reading Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:19.605036Z","iopub.execute_input":"2025-07-17T12:30:19.605503Z","iopub.status.idle":"2025-07-17T12:30:24.238204Z","shell.execute_reply.started":"2025-07-17T12:30:19.605482Z","shell.execute_reply":"2025-07-17T12:30:24.237559Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:24.239869Z","iopub.execute_input":"2025-07-17T12:30:24.240103Z","iopub.status.idle":"2025-07-17T12:30:24.244207Z","shell.execute_reply.started":"2025-07-17T12:30:24.240085Z","shell.execute_reply":"2025-07-17T12:30:24.243561Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:24.245153Z","iopub.execute_input":"2025-07-17T12:30:24.245373Z","iopub.status.idle":"2025-07-17T12:30:24.296818Z","shell.execute_reply.started":"2025-07-17T12:30:24.245356Z","shell.execute_reply":"2025-07-17T12:30:24.296109Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = train_df.drop([\"label\"], axis=1)\ny_train = train_df[\"label\"]\n\nX_test = test_df","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:24.297574Z","iopub.execute_input":"2025-07-17T12:30:24.297867Z","iopub.status.idle":"2025-07-17T12:30:24.404720Z","shell.execute_reply.started":"2025-07-17T12:30:24.297840Z","shell.execute_reply":"2025-07-17T12:30:24.404085Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(X_train.shape, X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:24.405470Z","iopub.execute_input":"2025-07-17T12:30:24.405705Z","iopub.status.idle":"2025-07-17T12:30:24.410124Z","shell.execute_reply.started":"2025-07-17T12:30:24.405686Z","shell.execute_reply":"2025-07-17T12:30:24.409304Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_plot = X_train.values.reshape(-1, 28, 28)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:24.410937Z","iopub.execute_input":"2025-07-17T12:30:24.411524Z","iopub.status.idle":"2025-07-17T12:30:24.424223Z","shell.execute_reply.started":"2025-07-17T12:30:24.411494Z","shell.execute_reply":"2025-07-17T12:30:24.423489Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef show_digits_data():\n    fig = plt.figure(figsize=(16,16))\n    for idx in range(15):\n        plt.subplot(5,5,idx+1)\n        plt.imshow(X_train_plot[idx], cmap=\"gray\")\n        plt.title(f\"Digits: {y_train[idx]}\")\n\nshow_digits_data()","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:24.425061Z","iopub.execute_input":"2025-07-17T12:30:24.425532Z","iopub.status.idle":"2025-07-17T12:30:26.690937Z","shell.execute_reply.started":"2025-07-17T12:30:24.425514Z","shell.execute_reply":"2025-07-17T12:30:26.690175Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"\nrand_number = random.randint(0, len(y_train))\nprint(f\"Digit: {y_train[rand_number]}\")\ndigit = X_train_plot[rand_number]\nplt.figure(figsize=(11, 11))\nsns.heatmap(\n    digit,\n    cmap='gray',         \n    annot=True,          \n    fmt='d',             \n    cbar=False,\n    linewidths=0.05,     \n    linecolor='#000000',   \n    square=True,         \n    annot_kws={\"size\": 6}  \n)\nplt.title(\"Digit as Heatmap\")\nplt.axis('off')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:26.693464Z","iopub.execute_input":"2025-07-17T12:30:26.693684Z","iopub.status.idle":"2025-07-17T12:30:28.165734Z","shell.execute_reply.started":"2025-07-17T12:30:26.693667Z","shell.execute_reply":"2025-07-17T12:30:28.164933Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"digit_range = np.arange(10)\n \nval = y_train.value_counts().index\ncnt = y_train.value_counts().values\nmycolors = ['red', 'blue', 'green', 'orange', 'brown', 'grey', 'pink', 'olive', 'deeppink', 'steelblue']\n\nplt.figure(figsize = (15, 7))\nplt.title(\"The number of digits in the data\", fontsize = 20)\nplt.xticks(range(10))\nplt.bar(val, cnt, color = mycolors);","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:28.166642Z","iopub.execute_input":"2025-07-17T12:30:28.166945Z","iopub.status.idle":"2025-07-17T12:30:28.381185Z","shell.execute_reply.started":"2025-07-17T12:30:28.166912Z","shell.execute_reply":"2025-07-17T12:30:28.380263Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"img_rows, img_cols = 28, 28\n\nnum_pixels = X_train.shape[1] \n\ninput_shape = (img_rows, img_cols)\nprint(input_shape)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:28.382107Z","iopub.execute_input":"2025-07-17T12:30:28.382946Z","iopub.status.idle":"2025-07-17T12:30:28.387766Z","shell.execute_reply.started":"2025-07-17T12:30:28.382916Z","shell.execute_reply":"2025-07-17T12:30:28.386921Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data Normalization [0, 1]\nX_train /= 255\nX_test /= 255\n\n# one-hot encoding for target column\ny_train = to_categorical(y_train)\n\n# | [0, 1, 2, ... , 9] | = 10\nnum_classes = y_train.shape[1]\n\n# Number of objects, vector size (28 * 28)\nprint(X_train.shape, X_test.shape, y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:28.388651Z","iopub.execute_input":"2025-07-17T12:30:28.388900Z","iopub.status.idle":"2025-07-17T12:30:28.508180Z","shell.execute_reply.started":"2025-07-17T12:30:28.388877Z","shell.execute_reply":"2025-07-17T12:30:28.507324Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state = 2, stratify= y_train)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:28.509060Z","iopub.execute_input":"2025-07-17T12:30:28.509592Z","iopub.status.idle":"2025-07-17T12:30:29.112735Z","shell.execute_reply.started":"2025-07-17T12:30:28.509562Z","shell.execute_reply":"2025-07-17T12:30:29.112111Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Functions to evaluate model while training","metadata":{}},{"cell_type":"code","source":"def f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    \n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:29.113576Z","iopub.execute_input":"2025-07-17T12:30:29.113864Z","iopub.status.idle":"2025-07-17T12:30:29.119779Z","shell.execute_reply.started":"2025-07-17T12:30:29.113834Z","shell.execute_reply":"2025-07-17T12:30:29.118951Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def draw_learning_curve(history, keys=['f1', 'loss']):\n    plt.figure(figsize=(20,8))\n    for i, key in enumerate(keys):\n        plt.subplot(1, 2, i + 1)\n        sns.lineplot(x = history.epoch, y = history.history[key])\n        sns.lineplot(x = history.epoch, y = history.history['val_' + key])\n        plt.title('Learning Curve')\n        plt.ylabel(key.title())\n        plt.xlabel('Epoch')\n#         plt.ylim(ylim)\n        plt.legend(['train', 'test'], loc='best')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:29.120701Z","iopub.execute_input":"2025-07-17T12:30:29.121575Z","iopub.status.idle":"2025-07-17T12:30:29.139487Z","shell.execute_reply.started":"2025-07-17T12:30:29.121550Z","shell.execute_reply":"2025-07-17T12:30:29.138849Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Callbacks","metadata":{}},{"cell_type":"code","source":"def callbacks(name): \n    return [ \n        EarlyStopping(monitor='val_loss', patience=8, restore_best_weights=True), \n        ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5, verbose=1), \n        ModelCheckpoint(f'../working/{name}.keras', save_best_only=True)\n    ]","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:29.140411Z","iopub.execute_input":"2025-07-17T12:30:29.141223Z","iopub.status.idle":"2025-07-17T12:30:29.154958Z","shell.execute_reply.started":"2025-07-17T12:30:29.141204Z","shell.execute_reply":"2025-07-17T12:30:29.154328Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"markdown","source":"## MLP_v1","metadata":{}},{"cell_type":"code","source":"def get_mlp():\n    \n    return Sequential([\n        #input layer is automatic generation by keras\n        \n        #hidden layer\n        Dense(512, input_dim = num_pixels, activation='relu'),\n        \n        #output layer\n        Dense(num_classes, activation='softmax')\n    ])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:29.155517Z","iopub.execute_input":"2025-07-17T12:30:29.155768Z","iopub.status.idle":"2025-07-17T12:30:29.169231Z","shell.execute_reply.started":"2025-07-17T12:30:29.155751Z","shell.execute_reply":"2025-07-17T12:30:29.168731Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = get_mlp()\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[f1])\nmodel.summary()\n\n\nlearning_history = model.fit(X_train, y_train,\n          batch_size = 1024, epochs = 40, verbose = 2, callbacks = callbacks('mlp_v1'),\n          validation_data=(X_val, y_val));","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:29.169831Z","iopub.execute_input":"2025-07-17T12:30:29.170028Z","iopub.status.idle":"2025-07-17T12:30:48.266735Z","shell.execute_reply.started":"2025-07-17T12:30:29.170012Z","shell.execute_reply":"2025-07-17T12:30:48.266138Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_val, y_val, verbose = 0)\nprint('Test loss: {}%'.format(score[0] * 100))\nprint('Test score: {}%'.format(score[1] * 100))\n\nprint(\"MLP Error: %.2f%%\" % (100 - score[1] * 100))\n\ndraw_learning_curve(learning_history)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:48.267513Z","iopub.execute_input":"2025-07-17T12:30:48.267704Z","iopub.status.idle":"2025-07-17T12:30:49.777320Z","shell.execute_reply.started":"2025-07-17T12:30:48.267689Z","shell.execute_reply":"2025-07-17T12:30:49.776415Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## MLP_v2","metadata":{}},{"cell_type":"code","source":"def get_mlpv2():\n    \n    return Sequential([\n        Dense(512, input_dim=num_pixels, activation='relu'),\n        Dropout(0.3),\n        Dense(256, activation='relu'),\n        Dropout(0.2),\n        Dense(128, activation='relu'),\n        Dense(num_classes, activation='softmax')\n    ])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:49.778330Z","iopub.execute_input":"2025-07-17T12:30:49.778695Z","iopub.status.idle":"2025-07-17T12:30:49.783432Z","shell.execute_reply.started":"2025-07-17T12:30:49.778668Z","shell.execute_reply":"2025-07-17T12:30:49.782778Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = get_mlpv2()\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[f1])\nmodel.summary()\n\nlearning_history = model.fit(X_train, y_train,\n          batch_size = 64, epochs = 40, verbose = 2, callbacks = callbacks('mlp_v2'),\n          validation_data=(X_val, y_val));","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:30:49.784333Z","iopub.execute_input":"2025-07-17T12:30:49.784653Z","iopub.status.idle":"2025-07-17T12:31:27.054267Z","shell.execute_reply.started":"2025-07-17T12:30:49.784632Z","shell.execute_reply":"2025-07-17T12:31:27.053680Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_val, y_val, verbose = 0)\nprint('Test loss: {}%'.format(score[0] * 100))\nprint('Test score: {}%'.format(score[1] * 100))\n\nprint(\"MLP Error: %.2f%%\" % (100 - score[1] * 100))\n\ndraw_learning_curve(learning_history)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:31:27.055138Z","iopub.execute_input":"2025-07-17T12:31:27.055388Z","iopub.status.idle":"2025-07-17T12:31:28.613578Z","shell.execute_reply.started":"2025-07-17T12:31:27.055371Z","shell.execute_reply":"2025-07-17T12:31:28.612574Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CNN_v1","metadata":{}},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:31:28.614567Z","iopub.execute_input":"2025-07-17T12:31:28.615578Z","iopub.status.idle":"2025-07-17T12:31:28.620913Z","shell.execute_reply.started":"2025-07-17T12:31:28.615544Z","shell.execute_reply":"2025-07-17T12:31:28.619993Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = X_train.values.reshape(-1, 28, 28, 1)\nX_val = X_val.values.reshape(-1, 28, 28, 1)\nX_test = X_test.values.reshape(-1, 28, 28, 1)\ninput_shape = (28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:31:28.621671Z","iopub.execute_input":"2025-07-17T12:31:28.621908Z","iopub.status.idle":"2025-07-17T12:31:28.639774Z","shell.execute_reply.started":"2025-07-17T12:31:28.621871Z","shell.execute_reply":"2025-07-17T12:31:28.638948Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_cnn():\n    return Sequential([\n        Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape = input_shape),\n        Conv2D(32, kernel_size=(3, 3), activation='relu' ),\n        MaxPool2D(pool_size=(2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),\n        Conv2D(64, kernel_size=(3, 3), activation='relu' ),\n        MaxPool2D(pool_size=(2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same' ),\n        Conv2D(128, kernel_size=(3, 3), activation='relu' ),\n        MaxPool2D(pool_size=(2, 2)),\n        Dropout(0.25),\n        \n        \n        Flatten(),\n        \n        Dense(256, activation='relu'),\n        Dropout(0.5),\n        Dense(num_classes, activation = \"softmax\")\n        \n    ])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:31:28.640633Z","iopub.execute_input":"2025-07-17T12:31:28.640887Z","iopub.status.idle":"2025-07-17T12:31:28.656482Z","shell.execute_reply.started":"2025-07-17T12:31:28.640860Z","shell.execute_reply":"2025-07-17T12:31:28.655875Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = get_cnn()\nmodel.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=[f1])\nmodel.summary()\n\nlearning_history = model.fit(X_train, y_train,\n          batch_size = 128,\n          epochs = 50,\n          verbose = 1,\n          callbacks = callbacks('cnn_v1'),\n          validation_data = (X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:31:28.657361Z","iopub.execute_input":"2025-07-17T12:31:28.657593Z","iopub.status.idle":"2025-07-17T12:32:42.121306Z","shell.execute_reply.started":"2025-07-17T12:31:28.657577Z","shell.execute_reply":"2025-07-17T12:32:42.120503Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test score:', score[1])\n\nprint(\"CNN Error: %.2f%%\" % (100-score[1]*100))\n\ndraw_learning_curve(learning_history)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:42.126720Z","iopub.execute_input":"2025-07-17T12:32:42.126951Z","iopub.status.idle":"2025-07-17T12:32:44.140710Z","shell.execute_reply.started":"2025-07-17T12:32:42.126932Z","shell.execute_reply":"2025-07-17T12:32:44.139961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_val)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:44.141651Z","iopub.execute_input":"2025-07-17T12:32:44.142529Z","iopub.status.idle":"2025-07-17T12:32:45.317545Z","shell.execute_reply.started":"2025-07-17T12:32:44.142502Z","shell.execute_reply":"2025-07-17T12:32:45.316912Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Functions to Analyze misclassified samples","metadata":{}},{"cell_type":"markdown","source":"Count the number of misclassified predictions for each label class","metadata":{}},{"cell_type":"code","source":"def check_wrong_predictions(model):\n    cnt_error = []\n    for idx, (a, b) in enumerate(zip(y_val, y_pred)):\n        if np.argmax(a) == np.argmax(b): continue\n        cnt_error.append( (np.argmax(a)) )\n    \n    cnt_error = np.unique(cnt_error, return_counts = True)\n    sns.set_style(\"darkgrid\")\n    plt.figure(figsize = (15, 7))\n    bar_plot = sns.barplot(x=cnt_error[0], y=cnt_error[1], palette=\"muted\")\n    plt.show()\n\ncheck_wrong_predictions(model)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:45.318492Z","iopub.execute_input":"2025-07-17T12:32:45.318744Z","iopub.status.idle":"2025-07-17T12:32:45.553542Z","shell.execute_reply.started":"2025-07-17T12:32:45.318726Z","shell.execute_reply":"2025-07-17T12:32:45.552725Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Visualize the Misclassified samples","metadata":{}},{"cell_type":"code","source":"cnt_ind = 1\nlist_idx = []\nX_val_plot = X_val.reshape( X_val.shape[:-1] )\nfig = plt.figure(figsize=(14, 14))\n\nfor idx, (a, b) in enumerate(zip(y_val, y_pred)):\n    if np.argmax(a) == np.argmax(b): continue\n    if (np.argmax(a) == 3 or np.argmax(a) == 4 or np.argmax(a) == 9):    \n        plt.subplot(5, 5, cnt_ind)\n        plt.imshow(X_val_plot[idx], cmap='gray', interpolation='none')\n        plt.title('y_true={0}\\ny_pred={1}\\n ind = {2}'.format(np.argmax(a), np.argmax(b), idx))\n        plt.tight_layout()\n        list_idx.append(idx)\n        cnt_ind += 1","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:45.554430Z","iopub.execute_input":"2025-07-17T12:32:45.555046Z","iopub.status.idle":"2025-07-17T12:32:51.423354Z","shell.execute_reply.started":"2025-07-17T12:32:45.555020Z","shell.execute_reply":"2025-07-17T12:32:51.422520Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\nFunction to check what probabilites the model predicted for each misclassified label class","metadata":{}},{"cell_type":"code","source":"def draw_output(idx_nums):\n    plt.figure(figsize = (25, 25))\n    plt.xticks( range(10) )\n    x = int(np.ceil(np.sqrt(len(idx_nums))))\n    cnt = 1\n    for ph in idx_nums:\n        plt.subplot(x, x, cnt)\n        curr_photo = y_val[ph]\n        \n        plt.xlim(0, 10)\n        plt.title(\"Digit: {0}\\n idx: {1} \".format(np.argmax(y_val[ph]), ph), fontsize = 10) \n        plt.bar(range(10), y_pred[ph])\n        \n        cnt += 1\n\n\ndraw_output(list_idx)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:51.424367Z","iopub.execute_input":"2025-07-17T12:32:51.424782Z","iopub.status.idle":"2025-07-17T12:32:54.729237Z","shell.execute_reply.started":"2025-07-17T12:32:51.424754Z","shell.execute_reply":"2025-07-17T12:32:54.728495Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Data Augmentation!","metadata":{}},{"cell_type":"markdown","source":"Image Generator expects converts float64 tensors into float32 so converting everything to float32 to not get any error","metadata":{}},{"cell_type":"code","source":"X_train = X_train.astype('float32')\ny_train = y_train.astype('float32')\nX_val = X_val.astype('float32')\ny_val = y_val.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:54.729988Z","iopub.execute_input":"2025-07-17T12:32:54.730221Z","iopub.status.idle":"2025-07-17T12:32:54.784955Z","shell.execute_reply.started":"2025-07-17T12:32:54.730198Z","shell.execute_reply":"2025-07-17T12:32:54.784108Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_aug = ImageDataGenerator(\n        featurewise_center = False,\n        samplewise_center = False,\n        featurewise_std_normalization = False, \n        samplewise_std_normalization = False,\n        zca_whitening = False,\n        horizontal_flip = False,\n        vertical_flip = False,\n        fill_mode = 'nearest',\n        rotation_range = 10,  \n        zoom_range = 0.1, \n        width_shift_range = 0.1, \n        height_shift_range = 0.1)\n        \n\ntrain_aug.fit(X_train)\ntrain_gen = train_aug.flow(X_train, y_train, batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:54.785803Z","iopub.execute_input":"2025-07-17T12:32:54.786070Z","iopub.status.idle":"2025-07-17T12:32:54.832751Z","shell.execute_reply.started":"2025-07-17T12:32:54.786044Z","shell.execute_reply":"2025-07-17T12:32:54.832138Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CNN_v2","metadata":{}},{"cell_type":"code","source":"def get_cnn_v2():\n    return Sequential([\n        Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape = input_shape),\n        Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same'),\n        BatchNormalization(),\n        MaxPool2D(pool_size=(2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),\n        Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same' ),\n        BatchNormalization(),\n        MaxPool2D(pool_size=(2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same' ),\n        Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same' ),\n        BatchNormalization(),\n        MaxPool2D(pool_size=(2, 2)),\n        Dropout(0.25),\n        \n        \n        Flatten(),\n          \n        Dense(512, activation='relu'),\n        BatchNormalization(),\n        Dropout(0.5),\n        \n        Dense(256, activation='relu'),\n        BatchNormalization(),\n        Dropout(0.4),\n        \n        Dense(64, activation='relu'),\n        BatchNormalization(),\n        Dropout(0.3),\n        \n        Dense(num_classes, activation = \"softmax\")\n        \n    ])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:54.833513Z","iopub.execute_input":"2025-07-17T12:32:54.833777Z","iopub.status.idle":"2025-07-17T12:32:54.840338Z","shell.execute_reply.started":"2025-07-17T12:32:54.833756Z","shell.execute_reply":"2025-07-17T12:32:54.839501Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = get_cnn_v2()\nmodel.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=[f1])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:54.841192Z","iopub.execute_input":"2025-07-17T12:32:54.841492Z","iopub.status.idle":"2025-07-17T12:32:55.017209Z","shell.execute_reply.started":"2025-07-17T12:32:54.841468Z","shell.execute_reply":"2025-07-17T12:32:55.016628Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_history = model.fit(train_gen, epochs = 100, \n                               steps_per_epoch = X_train.shape[0] // 64,\n                               validation_data = (X_val, y_val),\n                               callbacks = callbacks('cnn_v2'),\n                             )","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:32:55.017947Z","iopub.execute_input":"2025-07-17T12:32:55.018206Z","iopub.status.idle":"2025-07-17T12:38:36.638146Z","shell.execute_reply.started":"2025-07-17T12:32:55.018188Z","shell.execute_reply":"2025-07-17T12:38:36.637584Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = load_model('../working/cnn_v2.keras', custom_objects={\"f1\": f1})","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:36.639027Z","iopub.execute_input":"2025-07-17T12:38:36.639273Z","iopub.status.idle":"2025-07-17T12:38:37.194145Z","shell.execute_reply.started":"2025-07-17T12:38:36.639255Z","shell.execute_reply":"2025-07-17T12:38:37.193249Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test score:', score[1])\n\nprint(\"CNN Error: %.2f%%\" % (100-score[1]*100))","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:37.195024Z","iopub.execute_input":"2025-07-17T12:38:37.195221Z","iopub.status.idle":"2025-07-17T12:38:39.618091Z","shell.execute_reply.started":"2025-07-17T12:38:37.195206Z","shell.execute_reply":"2025-07-17T12:38:39.617409Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"draw_learning_curve(learning_history)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:39.618941Z","iopub.execute_input":"2025-07-17T12:38:39.619281Z","iopub.status.idle":"2025-07-17T12:38:40.194457Z","shell.execute_reply.started":"2025-07-17T12:38:39.619262Z","shell.execute_reply":"2025-07-17T12:38:40.193750Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_val)\ncheck_wrong_predictions(model)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:40.195380Z","iopub.execute_input":"2025-07-17T12:38:40.195951Z","iopub.status.idle":"2025-07-17T12:38:42.209202Z","shell.execute_reply.started":"2025-07-17T12:38:40.195923Z","shell.execute_reply":"2025-07-17T12:38:42.208323Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cnt_ind = 1\nlist_idx = []\nX_val_plot = X_val.reshape( X_val.shape[:-1] )\nfig = plt.figure(figsize=(14, 14))\n\nfor idx, (a, b) in enumerate(zip(y_val, y_pred)):\n    if np.argmax(a) == np.argmax(b): continue\n    if (np.argmax(a) == 3 or np.argmax(a) == 4 or np.argmax(a) == 9):    \n        plt.subplot(5, 5, cnt_ind)\n        plt.imshow(X_val_plot[idx], cmap='gray', interpolation='none')\n        plt.title('y_true={0}\\ny_pred={1}\\n ind = {2}'.format(np.argmax(a), np.argmax(b), idx))\n        plt.tight_layout()\n        list_idx.append(idx)\n        cnt_ind += 1","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:42.210146Z","iopub.execute_input":"2025-07-17T12:38:42.210501Z","iopub.status.idle":"2025-07-17T12:38:44.007236Z","shell.execute_reply.started":"2025-07-17T12:38:42.210471Z","shell.execute_reply":"2025-07-17T12:38:44.006361Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"draw_output(list_idx)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:44.007978Z","iopub.execute_input":"2025-07-17T12:38:44.008179Z","iopub.status.idle":"2025-07-17T12:38:45.458567Z","shell.execute_reply.started":"2025-07-17T12:38:44.008161Z","shell.execute_reply":"2025-07-17T12:38:45.457590Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Targeted Learning\n### Create New Data with only Misclassified examples and then Retrain the CNN_V2\n\nAs you can see the cnn_v2 did perform better than rest of the models but still it is misclassifying for some images.\n\nBut these images are hard to classify even by a human. Lets try to Fine Tune our model using these misclassified samples. ","metadata":{}},{"cell_type":"code","source":"cnn_best_model = load_model('../working/cnn_v2.keras', custom_objects={\"f1\": f1})","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:45.459516Z","iopub.execute_input":"2025-07-17T12:38:45.459789Z","iopub.status.idle":"2025-07-17T12:38:45.928237Z","shell.execute_reply.started":"2025-07-17T12:38:45.459769Z","shell.execute_reply":"2025-07-17T12:38:45.927677Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Get the prediction labels and true labels","metadata":{}},{"cell_type":"code","source":"y_pred_probs = cnn_best_model.predict(X_val)\ny_pred_classes = np.argmax(y_pred_probs, axis=1)\ny_true = np.argmax(y_val, axis=1)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:45.928916Z","iopub.execute_input":"2025-07-17T12:38:45.929121Z","iopub.status.idle":"2025-07-17T12:38:47.671154Z","shell.execute_reply.started":"2025-07-17T12:38:45.929105Z","shell.execute_reply":"2025-07-17T12:38:47.670515Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Get the misclassified images/samples \n\nAs you can see we got 23 Images that are not classified correctly.","metadata":{}},{"cell_type":"code","source":"incorrect_indices = np.where(y_pred_classes!=y_true)\nprint(incorrect_indices)\nprint(len(incorrect_indices[0]))","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.671977Z","iopub.execute_input":"2025-07-17T12:38:47.672209Z","iopub.status.idle":"2025-07-17T12:38:47.677280Z","shell.execute_reply.started":"2025-07-17T12:38:47.672191Z","shell.execute_reply":"2025-07-17T12:38:47.676432Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_misclassified = X_val[incorrect_indices]\ny_misclassified = y_val[incorrect_indices]","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.678088Z","iopub.execute_input":"2025-07-17T12:38:47.678291Z","iopub.status.idle":"2025-07-17T12:38:47.698019Z","shell.execute_reply.started":"2025-07-17T12:38:47.678276Z","shell.execute_reply":"2025-07-17T12:38:47.697492Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Now lets try 2 Different Approaches!\n1) Append the misclassified Augmentated data to the earlier augmented data and then re Augment the Whole data! (Not Feasible)\n   \n2) Use only the misclassifed data add it with original train data and then Augment it i.e. only one time Augmentation!","metadata":{}},{"cell_type":"markdown","source":"## 1st Approach","metadata":{}},{"cell_type":"markdown","source":"Create the Misclassified Gen Object","metadata":{}},{"cell_type":"code","source":"\nmisclassified_gen = ImageDataGenerator(\n    rotation_range=10,\n    zoom_range=0.1,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n)\n\nmisclassified_gen.fit(X_misclassified)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.698716Z","iopub.execute_input":"2025-07-17T12:38:47.698883Z","iopub.status.idle":"2025-07-17T12:38:47.716310Z","shell.execute_reply.started":"2025-07-17T12:38:47.698869Z","shell.execute_reply":"2025-07-17T12:38:47.715357Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Create samples. here we are creating 320 samples. You can do more if you want!","metadata":{}},{"cell_type":"code","source":"augmented_images = []\naugmented_labels = []\n\nn_batches = 10  # generate 10 batches and each batch is of size 32 i.e. 320 samples\n\nfor _ in range(n_batches):\n    X_aug, y_aug = next(misclassified_gen.flow(X_misclassified, y_misclassified, batch_size=32, shuffle=True))\n    augmented_images.append(X_aug)\n    augmented_labels.append(y_aug)\n\n# Stack all batches together\naugmented_images = np.vstack(augmented_images)\naugmented_labels = np.vstack(augmented_labels)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.717354Z","iopub.execute_input":"2025-07-17T12:38:47.717639Z","iopub.status.idle":"2025-07-17T12:38:47.777544Z","shell.execute_reply.started":"2025-07-17T12:38:47.717616Z","shell.execute_reply":"2025-07-17T12:38:47.776964Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_augmented = np.concatenate([X_train, augmented_images])\ny_train_augmented = np.concatenate([y_train, augmented_labels])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.778364Z","iopub.execute_input":"2025-07-17T12:38:47.778848Z","iopub.status.idle":"2025-07-17T12:38:47.847526Z","shell.execute_reply.started":"2025-07-17T12:38:47.778822Z","shell.execute_reply":"2025-07-17T12:38:47.846632Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Use the earlier Augmentation Object i.e. train_aug","metadata":{}},{"cell_type":"markdown","source":"Note:- Here , If you face any Error. try to reduce the batch size. The error is due to creating Augmentations which are generaly stored in memory and not storage!","metadata":{}},{"cell_type":"code","source":"train_gen = train_aug.flow(X_train_augmented, y_train_augmented, batch_size=64, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.848423Z","iopub.execute_input":"2025-07-17T12:38:47.848714Z","iopub.status.idle":"2025-07-17T12:38:47.852781Z","shell.execute_reply.started":"2025-07-17T12:38:47.848691Z","shell.execute_reply":"2025-07-17T12:38:47.852180Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = get_cnn_v2()\nmodel.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=[f1])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:47.853492Z","iopub.execute_input":"2025-07-17T12:38:47.853709Z","iopub.status.idle":"2025-07-17T12:38:48.492652Z","shell.execute_reply.started":"2025-07-17T12:38:47.853693Z","shell.execute_reply":"2025-07-17T12:38:48.492022Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_history = model.fit(train_gen, epochs = 100, \n                               steps_per_epoch = X_train.shape[0] // 64,\n                               validation_data = (X_val, y_val),\n                               callbacks = callbacks('ccn_v2_aug_2times'),\n                             )","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:38:48.493287Z","iopub.execute_input":"2025-07-17T12:38:48.493524Z","iopub.status.idle":"2025-07-17T12:44:06.594653Z","shell.execute_reply.started":"2025-07-17T12:38:48.493508Z","shell.execute_reply":"2025-07-17T12:44:06.593909Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = load_model('../working/ccn_v2_aug_2times.keras', custom_objects={\"f1\": f1})","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:44:06.595729Z","iopub.execute_input":"2025-07-17T12:44:06.596083Z","iopub.status.idle":"2025-07-17T12:44:07.100127Z","shell.execute_reply.started":"2025-07-17T12:44:06.596062Z","shell.execute_reply":"2025-07-17T12:44:07.099579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test score:', score[1])\n\nprint(\"CNN Error: %.2f%%\" % (100-score[1]*100))","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:44:07.100759Z","iopub.execute_input":"2025-07-17T12:44:07.100959Z","iopub.status.idle":"2025-07-17T12:44:09.312100Z","shell.execute_reply.started":"2025-07-17T12:44:07.100942Z","shell.execute_reply":"2025-07-17T12:44:09.311494Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"draw_learning_curve(learning_history)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:44:09.312865Z","iopub.execute_input":"2025-07-17T12:44:09.313163Z","iopub.status.idle":"2025-07-17T12:44:09.846496Z","shell.execute_reply.started":"2025-07-17T12:44:09.313133Z","shell.execute_reply":"2025-07-17T12:44:09.845737Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_val)\ncheck_wrong_predictions(model)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:44:09.847357Z","iopub.execute_input":"2025-07-17T12:44:09.847858Z","iopub.status.idle":"2025-07-17T12:44:11.660290Z","shell.execute_reply.started":"2025-07-17T12:44:09.847833Z","shell.execute_reply":"2025-07-17T12:44:11.659471Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cnt_ind = 1\nlist_idx = []\nX_val_plot = X_val.reshape( X_val.shape[:-1] )\nfig = plt.figure(figsize=(14, 14))\n\nfor idx, (a, b) in enumerate(zip(y_val, y_pred)):\n    if np.argmax(a) == np.argmax(b): continue\n    if (np.argmax(a) == 1 or np.argmax(a) == 2 or np.argmax(a) == 3 or np.argmax(a) == 4 or np.argmax(a) == 9):    \n        plt.subplot(5, 5, cnt_ind)\n        plt.imshow(X_val_plot[idx], cmap='gray', interpolation='none')\n        plt.title('y_true={0}\\ny_pred={1}\\n ind = {2}'.format(np.argmax(a), np.argmax(b), idx))\n        plt.tight_layout()\n        list_idx.append(idx)\n        cnt_ind += 1","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:44:11.661268Z","iopub.execute_input":"2025-07-17T12:44:11.661535Z","iopub.status.idle":"2025-07-17T12:44:13.265313Z","shell.execute_reply.started":"2025-07-17T12:44:11.661517Z","shell.execute_reply":"2025-07-17T12:44:13.264649Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"draw_output(list_idx)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T12:44:13.266090Z","iopub.execute_input":"2025-07-17T12:44:13.266471Z","iopub.status.idle":"2025-07-17T12:44:14.583746Z","shell.execute_reply.started":"2025-07-17T12:44:13.266427Z","shell.execute_reply":"2025-07-17T12:44:14.583006Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2nd Approach","metadata":{}},{"cell_type":"markdown","source":"Add the misclassified samples to original data without augmentation","metadata":{}},{"cell_type":"code","source":"X_train = X_train.astype('float32')\ny_train = y_train.astype('float32')\nX_val = X_val.astype('float32')\ny_val = y_val.astype('float32')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T13:01:44.949363Z","iopub.execute_input":"2025-07-17T13:01:44.950091Z","iopub.status.idle":"2025-07-17T13:01:45.008593Z","shell.execute_reply.started":"2025-07-17T13:01:44.950067Z","shell.execute_reply":"2025-07-17T13:01:45.007723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_misclassifed = np.concatenate([X_train, X_misclassified])\ny_train_misclassified = np.concatenate([y_train, y_misclassified])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:01:46.603178Z","iopub.execute_input":"2025-07-17T13:01:46.603663Z","iopub.status.idle":"2025-07-17T13:01:46.675900Z","shell.execute_reply.started":"2025-07-17T13:01:46.603639Z","shell.execute_reply":"2025-07-17T13:01:46.674924Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Note:- Here , If you face any Error. try to reduce the batch size. The error is due to creating Augmentations which are generaly stored in memory and not storage!","metadata":{}},{"cell_type":"code","source":"train_gen = train_aug.flow(X_train_misclassifed, y_train_misclassified, batch_size=64, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:01:47.920557Z","iopub.execute_input":"2025-07-17T13:01:47.921058Z","iopub.status.idle":"2025-07-17T13:01:47.924408Z","shell.execute_reply.started":"2025-07-17T13:01:47.921036Z","shell.execute_reply":"2025-07-17T13:01:47.923790Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = get_cnn_v2()\nmodel.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=[f1])","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:01:48.506133Z","iopub.execute_input":"2025-07-17T13:01:48.506811Z","iopub.status.idle":"2025-07-17T13:01:48.641776Z","shell.execute_reply.started":"2025-07-17T13:01:48.506787Z","shell.execute_reply":"2025-07-17T13:01:48.641176Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_history = model.fit(train_gen, epochs = 100, \n                               steps_per_epoch = X_train.shape[0] // 64,\n                               validation_data = (X_val, y_val),\n                               callbacks = callbacks('cnn_best_aug_1time'),\n                             )","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:01:49.077025Z","iopub.execute_input":"2025-07-17T13:01:49.077611Z","iopub.status.idle":"2025-07-17T13:09:35.296505Z","shell.execute_reply.started":"2025-07-17T13:01:49.077590Z","shell.execute_reply":"2025-07-17T13:09:35.295756Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = load_model('../working/cnn_best_aug_1time.keras', custom_objects={\"f1\": f1})\n\nscore = model.evaluate(X_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test score:', score[1])\n\nprint(\"CNN Error: %.2f%%\" % (100-score[1]*100))","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:32:52.168293Z","iopub.execute_input":"2025-07-17T13:32:52.168951Z","iopub.status.idle":"2025-07-17T13:32:55.552365Z","shell.execute_reply.started":"2025-07-17T13:32:52.168923Z","shell.execute_reply":"2025-07-17T13:32:55.551646Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"draw_learning_curve(learning_history)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:03.782352Z","iopub.execute_input":"2025-07-17T13:33:03.782622Z","iopub.status.idle":"2025-07-17T13:33:04.347409Z","shell.execute_reply.started":"2025-07-17T13:33:03.782604Z","shell.execute_reply":"2025-07-17T13:33:04.346641Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_val)\ncheck_wrong_predictions(model)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:09.617570Z","iopub.execute_input":"2025-07-17T13:33:09.618222Z","iopub.status.idle":"2025-07-17T13:33:11.568151Z","shell.execute_reply.started":"2025-07-17T13:33:09.618201Z","shell.execute_reply":"2025-07-17T13:33:11.567377Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cnt_ind = 1\nlist_idx = []\nX_val_plot = X_val.reshape( X_val.shape[:-1] )\nfig = plt.figure(figsize=(14, 14))\n\nfor idx, (a, b) in enumerate(zip(y_val, y_pred)):\n    if np.argmax(a) == np.argmax(b): continue\n    if (np.argmax(a) == 1 or np.argmax(a) == 4 or np.argmax(a) == 6 or np.argmax(a) == 9):    \n        plt.subplot(5, 5, cnt_ind)\n        plt.imshow(X_val_plot[idx], cmap='gray', interpolation='none')\n        plt.title('y_true={0}\\ny_pred={1}\\n ind = {2}'.format(np.argmax(a), np.argmax(b), idx))\n        plt.tight_layout()\n        list_idx.append(idx)\n        cnt_ind += 1","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:17.680522Z","iopub.execute_input":"2025-07-17T13:33:17.681231Z","iopub.status.idle":"2025-07-17T13:33:19.133948Z","shell.execute_reply.started":"2025-07-17T13:33:17.681204Z","shell.execute_reply":"2025-07-17T13:33:19.133166Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"draw_output(list_idx)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:19.135287Z","iopub.execute_input":"2025-07-17T13:33:19.136040Z","iopub.status.idle":"2025-07-17T13:33:20.453116Z","shell.execute_reply.started":"2025-07-17T13:33:19.136010Z","shell.execute_reply":"2025-07-17T13:33:20.452271Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"the final best model **cnn_best_aug_1time.keras** achieved this results:\n\n##### Test loss: 0.004026979673653841\n##### Test score: 0.9988163113594055\n##### CNN Error: 0.12%\n","metadata":{}},{"cell_type":"markdown","source":"Here is the confusion matrix and classification report","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, classification_report\n\ny_pred_labels = np.argmax(y_pred, axis=1)\n\ncm = confusion_matrix(y_true, y_pred_labels)\n\nplt.figure(figsize=(8,6))\nsns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\nplt.xlabel(\"Predicted Label\")\nplt.ylabel(\"True Label\")\nplt.title(\"Confusion Matrix\")\nplt.show()\n\n\n\nprint(classification_report(y_true, y_pred_labels))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T13:33:23.009029Z","iopub.execute_input":"2025-07-17T13:33:23.009321Z","iopub.status.idle":"2025-07-17T13:33:23.472194Z","shell.execute_reply.started":"2025-07-17T13:33:23.009298Z","shell.execute_reply":"2025-07-17T13:33:23.471509Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Note:- you can use this Approach multiple times and check how our modelling is performing. is it overffiting or model's performance is not increasing at all.\n\n### Another approach would be increasing the data for label 9 as it has ~1k less samples that label 0 and 1\n\n### If this approach isn't working then try to use models such as small RestNet or MobileNet. They might work!","metadata":{}},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"output = model.predict(X_test)\noutput = np.argmax(output, axis=1)\n\noutput = pd.Series(output, name='Label')\nsubmission = pd.concat([pd.Series(range(1, len(output) + 1), name='ImageId'), output], axis=1)\n\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:39.007523Z","iopub.execute_input":"2025-07-17T13:33:39.008226Z","iopub.status.idle":"2025-07-17T13:33:41.755362Z","shell.execute_reply.started":"2025-07-17T13:33:39.008202Z","shell.execute_reply":"2025-07-17T13:33:41.754740Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(submission.shape)         # Should be (28000, 2)\nprint(submission.dtypes)        # ImageId: int64, Label: int64\nprint(submission.tail())        # Ensure no blanks or missing numbers\nsubmission.isnull().sum()       # Should be all zeros\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T13:34:27.113714Z","iopub.execute_input":"2025-07-17T13:34:27.113968Z","iopub.status.idle":"2025-07-17T13:34:27.123567Z","shell.execute_reply.started":"2025-07-17T13:34:27.113951Z","shell.execute_reply":"2025-07-17T13:34:27.122613Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***** Extras *****","metadata":{}},{"cell_type":"markdown","source":"## Occlusion Sensitivity","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n\nX_train = train.drop(labels = [\"label\"], axis = 1)\ny_train = train['label']\ny_train = to_categorical(y_train)\n\nX_train /= 255\nX_train = X_train.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:47.480646Z","iopub.execute_input":"2025-07-17T13:33:47.480923Z","iopub.status.idle":"2025-07-17T13:33:49.430731Z","shell.execute_reply.started":"2025-07-17T13:33:47.480904Z","shell.execute_reply":"2025-07-17T13:33:49.430083Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(X_train[0].reshape(28, 28), cmap='gray');","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:49.431976Z","iopub.execute_input":"2025-07-17T13:33:49.432192Z","iopub.status.idle":"2025-07-17T13:33:49.709772Z","shell.execute_reply.started":"2025-07-17T13:33:49.432175Z","shell.execute_reply":"2025-07-17T13:33:49.708878Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = load_model('../working/cnn_best_aug_1time.keras', custom_objects={\"f1\": f1})","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:49.710565Z","iopub.execute_input":"2025-07-17T13:33:49.710790Z","iopub.status.idle":"2025-07-17T13:33:50.176988Z","shell.execute_reply.started":"2025-07-17T13:33:49.710772Z","shell.execute_reply":"2025-07-17T13:33:50.176412Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_grey_patch(image, top_left_x, top_left_y, patch_size):\n    patched_image = np.array(image, copy=True)\n    patched_image[top_left_y:top_left_y + patch_size, top_left_x:top_left_x + patch_size, :] = 0\n\n    return patched_image\n\n\nimg = X_train[0]\n\nPATCH_SIZE = 4\nsensitivity_map = np.zeros((img.shape[0], img.shape[0]))\n\nfor top_left_x in range(0, img.shape[0], PATCH_SIZE):\n    for top_left_y in range(0, img.shape[1], PATCH_SIZE):\n        patched_image = apply_grey_patch(img, top_left_x, top_left_y, PATCH_SIZE)\n        \n        \n        predicted_classes = model.predict(np.array([patched_image]))[0]\n        confidence = predicted_classes[1]\n        \n        sensitivity_map[\n            top_left_y:top_left_y + PATCH_SIZE,\n            top_left_x:top_left_x + PATCH_SIZE,\n        ] = confidence\n","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:50.178114Z","iopub.execute_input":"2025-07-17T13:33:50.178378Z","iopub.status.idle":"2025-07-17T13:33:54.846634Z","shell.execute_reply.started":"2025-07-17T13:33:50.178348Z","shell.execute_reply":"2025-07-17T13:33:54.845941Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(sensitivity_map, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2025-07-17T13:33:54.847984Z","iopub.execute_input":"2025-07-17T13:33:54.848257Z","iopub.status.idle":"2025-07-17T13:33:55.088591Z","shell.execute_reply.started":"2025-07-17T13:33:54.848227Z","shell.execute_reply":"2025-07-17T13:33:55.087835Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}