{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np\nimport pandas as pd\n\nnp.random.seed(0) \nimport random\n\nimport tensorflow.keras as keras\nfrom tensorflow.keras.models import Sequential, load_model\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten\nfrom tensorflow.keras.layers import Conv2D, MaxPool2D, BatchNormalization\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-19T14:17:01.230058Z","iopub.execute_input":"2023-11-19T14:17:01.230497Z","iopub.status.idle":"2023-11-19T14:17:01.244855Z","shell.execute_reply.started":"2023-11-19T14:17:01.230462Z","shell.execute_reply":"2023-11-19T14:17:01.243456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")\n\n\nX_train = train.drop(labels = [\"label\"], axis = 1)\ny_train = train['label']\n\nX_test = test\n\nprint(X_train.shape, X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:17:09.632692Z","iopub.execute_input":"2023-11-19T14:17:09.633167Z","iopub.status.idle":"2023-11-19T14:17:17.431014Z","shell.execute_reply.started":"2023-11-19T14:17:09.633131Z","shell.execute_reply":"2023-11-19T14:17:17.429887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_plot = X_train.values.reshape(-1, 28, 28)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:17:57.521314Z","iopub.execute_input":"2023-11-19T14:17:57.521825Z","iopub.status.idle":"2023-11-19T14:17:57.527900Z","shell.execute_reply.started":"2023-11-19T14:17:57.521788Z","shell.execute_reply":"2023-11-19T14:17:57.526657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Show_example_digits(mono = 'gray'):\n    fig = plt.figure(figsize = (16, 16))\n    for idx in range(15):\n        plt.subplot(5, 5,idx+1)\n        plt.imshow(X_train_plot[idx], cmap = mono)\n        plt.title(\"Digit {}\".format(y_train[idx]))\n        \n    plt.tight_layout()\n    \nShow_example_digits()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:18:11.132331Z","iopub.execute_input":"2023-11-19T14:18:11.133533Z","iopub.status.idle":"2023-11-19T14:18:15.352007Z","shell.execute_reply.started":"2023-11-19T14:18:11.133486Z","shell.execute_reply":"2023-11-19T14:18:15.349107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_digit(digit, dem = 28, font_size = 12):\n    max_ax = font_size * dem\n    \n    fig = plt.figure(figsize=(13, 13))\n    plt.xlim([0, max_ax])\n    plt.ylim([0, max_ax])\n    plt.axis('off')\n    black = '#000000'\n    \n    for idx in range(dem):\n        for jdx in range(dem):\n\n            t = plt.text(idx * font_size, max_ax - jdx*font_size, digit[jdx][idx], fontsize = font_size, color = black)\n            c = digit[jdx][idx] / 255.\n            t.set_bbox(dict(facecolor=(c, c, c), alpha = 0.5, edgecolor = 'black'))\n            \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:20:44.619256Z","iopub.execute_input":"2023-11-19T14:20:44.619817Z","iopub.status.idle":"2023-11-19T14:20:44.630834Z","shell.execute_reply.started":"2023-11-19T14:20:44.619778Z","shell.execute_reply":"2023-11-19T14:20:44.629162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_number = random.randint(0, len(y_train))\nprint(y_train[rand_number])\nplot_digit(X_train_plot[rand_number])","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:20:50.087882Z","iopub.execute_input":"2023-11-19T14:20:50.088395Z","iopub.status.idle":"2023-11-19T14:20:52.346307Z","shell.execute_reply.started":"2023-11-19T14:20:50.088339Z","shell.execute_reply":"2023-11-19T14:20:52.345461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"digit_range = np.arange(10)\n\nval = y_train.value_counts().index\ncnt = y_train.value_counts().values\nmycolors = ['red', 'blue', 'green', 'orange', 'pink', 'yellow', 'pink', 'lightblue', 'deeppink', 'steelblue']\n\nplt.figure(figsize = (15, 7))\nplt.title(\"The number of digits in the data\", fontsize = 20)\nplt.xticks(range(10))\nplt.bar(val, cnt, color = mycolors);","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:23:53.109875Z","iopub.execute_input":"2023-11-19T14:23:53.110319Z","iopub.status.idle":"2023-11-19T14:23:53.519636Z","shell.execute_reply.started":"2023-11-19T14:23:53.110286Z","shell.execute_reply":"2023-11-19T14:23:53.518302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_rows, img_cols = 28, 28\n\nnum_pixels = X_train.shape[1] \n\ninput_shape = (img_rows, img_cols)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:24:13.885827Z","iopub.execute_input":"2023-11-19T14:24:13.886322Z","iopub.status.idle":"2023-11-19T14:24:13.893315Z","shell.execute_reply.started":"2023-11-19T14:24:13.886283Z","shell.execute_reply":"2023-11-19T14:24:13.891771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train /= 255\nX_test /= 255\n\n# one-hot encoding for target column\ny_train = to_categorical(y_train)\n\n# | [0, 1, 2, ... , 9] | = 10\nnum_classes = y_train.shape[1]\n\n# Number of objects, vector size (28 * 28)\nprint(X_train.shape, X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:24:27.754190Z","iopub.execute_input":"2023-11-19T14:24:27.754836Z","iopub.status.idle":"2023-11-19T14:24:27.926980Z","shell.execute_reply.started":"2023-11-19T14:24:27.754784Z","shell.execute_reply":"2023-11-19T14:24:27.925747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state = 2, stratify=y_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:24:42.600512Z","iopub.execute_input":"2023-11-19T14:24:42.601810Z","iopub.status.idle":"2023-11-19T14:24:43.492255Z","shell.execute_reply.started":"2023-11-19T14:24:42.601752Z","shell.execute_reply":"2023-11-19T14:24:43.490681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    \n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:25:01.041008Z","iopub.execute_input":"2023-11-19T14:25:01.041478Z","iopub.status.idle":"2023-11-19T14:25:01.051803Z","shell.execute_reply.started":"2023-11-19T14:25:01.041444Z","shell.execute_reply":"2023-11-19T14:25:01.050285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_learning_curve(history, keys=['f1', 'loss']):\n    plt.figure(figsize=(20,8))\n    for i, key in enumerate(keys):\n        plt.subplot(1, 2, i + 1)\n        sns.lineplot(x = history.epoch, y = history.history[key])\n        sns.lineplot(x = history.epoch, y = history.history['val_' + key])\n        plt.title('Learning Curve')\n        plt.ylabel(key.title())\n        plt.xlabel('Epoch')\n#         plt.ylim(ylim)\n        plt.legend(['train', 'test'], loc='best')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:25:25.051747Z","iopub.execute_input":"2023-11-19T14:25:25.052169Z","iopub.status.idle":"2023-11-19T14:25:25.061635Z","shell.execute_reply.started":"2023-11-19T14:25:25.052137Z","shell.execute_reply":"2023-11-19T14:25:25.060128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def callbacks(name): \n    return [ \n        EarlyStopping(monitor = 'loss', patience = 6), \n        ReduceLROnPlateau(monitor = 'loss', patience = 3), \n        ModelCheckpoint(f'../working/{name}.hdf5', save_best_only=True) \n    ]","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:25:39.216761Z","iopub.execute_input":"2023-11-19T14:25:39.217189Z","iopub.status.idle":"2023-11-19T14:25:39.224140Z","shell.execute_reply.started":"2023-11-19T14:25:39.217157Z","shell.execute_reply":"2023-11-19T14:25:39.222787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mlp():\n    \n    return Sequential([\n        \n        Dense(512, input_dim = num_pixels, activation='relu'),\n       \n        Dense(num_classes, activation='softmax')\n    ])","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:26:15.307594Z","iopub.execute_input":"2023-11-19T14:26:15.308080Z","iopub.status.idle":"2023-11-19T14:26:15.315871Z","shell.execute_reply.started":"2023-11-19T14:26:15.308042Z","shell.execute_reply":"2023-11-19T14:26:15.313665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_mlp()\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[f1])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:26:34.915004Z","iopub.execute_input":"2023-11-19T14:26:34.915538Z","iopub.status.idle":"2023-11-19T14:26:35.142075Z","shell.execute_reply.started":"2023-11-19T14:26:34.915499Z","shell.execute_reply":"2023-11-19T14:26:35.140554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mlpv2():\n    \n    return Sequential([\n        Dense(512, input_dim=num_pixels, activation='relu'),\n        Dropout(0.3),\n        Dense(256, activation='relu'),\n        Dropout(0.2),\n        Dense(128, activation='relu'),\n        Dense(num_classes, activation='softmax')\n    ])","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:27:35.337438Z","iopub.execute_input":"2023-11-19T14:27:35.337868Z","iopub.status.idle":"2023-11-19T14:27:35.345736Z","shell.execute_reply.started":"2023-11-19T14:27:35.337831Z","shell.execute_reply":"2023-11-19T14:27:35.344323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_mlpv2()\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[f1])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:27:53.867231Z","iopub.execute_input":"2023-11-19T14:27:53.867756Z","iopub.status.idle":"2023-11-19T14:27:54.015117Z","shell.execute_reply.started":"2023-11-19T14:27:53.867718Z","shell.execute_reply":"2023-11-19T14:27:54.013847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model.evaluate(X_val, y_val, verbose = 0)\nprint('Test loss: {}%'.format(score[0] * 100))\nprint('Test score: {}%'.format(score[1] * 100))\n\nprint(\"MLP Error: %.2f%%\" % (100 - score[1] * 100))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T14:28:38.717439Z","iopub.execute_input":"2023-11-19T14:28:38.717915Z","iopub.status.idle":"2023-11-19T14:28:39.522530Z","shell.execute_reply.started":"2023-11-19T14:28:38.717882Z","shell.execute_reply":"2023-11-19T14:28:39.521034Z"},"trusted":true},"execution_count":null,"outputs":[]}]}