{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Digit Recognizer using Artificial Neural Networks","metadata":{}},{"cell_type":"markdown","source":"### Import librairies","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense \nfrom tensorflow.keras import layers\nfrom tensorflow.keras.losses import SparseCategoricalCrossentropy","metadata":{"execution":{"iopub.status.busy":"2023-05-26T08:55:55.395566Z","iopub.execute_input":"2023-05-26T08:55:55.396194Z","iopub.status.idle":"2023-05-26T08:56:07.224842Z","shell.execute_reply.started":"2023-05-26T08:55:55.396141Z","shell.execute_reply":"2023-05-26T08:56:07.223227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import the data","metadata":{}},{"cell_type":"code","source":"# Digital number in .cvs format\n\ntrain = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nX_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-05-26T08:56:15.938687Z","iopub.execute_input":"2023-05-26T08:56:15.939596Z","iopub.status.idle":"2023-05-26T08:56:22.696640Z","shell.execute_reply.started":"2023-05-26T08:56:15.939551Z","shell.execute_reply":"2023-05-26T08:56:22.695413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploring the data","metadata":{}},{"cell_type":"code","source":"# Basic informations about the data\nprint(\"Dataset columns: \", train.columns)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-26T08:58:11.840012Z","iopub.execute_input":"2023-05-26T08:58:11.841805Z","iopub.status.idle":"2023-05-26T08:58:11.850947Z","shell.execute_reply.started":"2023-05-26T08:58:11.841728Z","shell.execute_reply":"2023-05-26T08:58:11.849355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separate features from the target (\"label\" column here)\nX = train.drop('label', axis = 1)\ny = train.label                   #target attribute","metadata":{"execution":{"iopub.status.busy":"2023-05-26T08:58:27.591551Z","iopub.execute_input":"2023-05-26T08:58:27.592179Z","iopub.status.idle":"2023-05-26T08:58:27.744494Z","shell.execute_reply.started":"2023-05-26T08:58:27.592126Z","shell.execute_reply":"2023-05-26T08:58:27.743163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalization & split the data","metadata":{}},{"cell_type":"code","source":"# Features values must range from 0 to 1\nX = X / 255.0    \nX_test = X_test/255.0","metadata":{"execution":{"iopub.status.busy":"2023-05-26T08:59:40.741408Z","iopub.execute_input":"2023-05-26T08:59:40.741873Z","iopub.status.idle":"2023-05-26T08:59:40.859323Z","shell.execute_reply.started":"2023-05-26T08:59:40.741834Z","shell.execute_reply":"2023-05-26T08:59:40.858167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the data into training set and validation set\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, train_size = 0.80, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:00:01.811251Z","iopub.execute_input":"2023-05-26T09:00:01.811830Z","iopub.status.idle":"2023-05-26T09:00:02.157323Z","shell.execute_reply.started":"2023-05-26T09:00:01.811775Z","shell.execute_reply":"2023-05-26T09:00:02.155886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\n\nprint(f\"X_val shape: {X_val.shape}\")\nprint(f\"y_val shape: {y_val.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:00:07.751731Z","iopub.execute_input":"2023-05-26T09:00:07.752286Z","iopub.status.idle":"2023-05-26T09:00:07.760462Z","shell.execute_reply.started":"2023-05-26T09:00:07.752233Z","shell.execute_reply":"2023-05-26T09:00:07.759153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualization","metadata":{}},{"cell_type":"code","source":"# Resize the data to it's original form to visualize\nX_train_reshaped = X_train.values.reshape(-1, 28, 28)\nplt.imshow(X_train_reshaped[10])","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:00:55.030237Z","iopub.execute_input":"2023-05-26T09:00:55.031006Z","iopub.status.idle":"2023-05-26T09:00:55.360011Z","shell.execute_reply.started":"2023-05-26T09:00:55.030926Z","shell.execute_reply":"2023-05-26T09:00:55.358193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize the first 10 images and their label\n\nplt.figure(figsize=(10, 15))\nfor i in range(10):\n    plt.subplot(5, 5, i+1)\n    plt.grid(False)\n    plt.imshow(X_train_reshaped[i])\n    plt.xlabel(y_train.iloc[i])","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:01:33.656055Z","iopub.execute_input":"2023-05-26T09:01:33.656570Z","iopub.status.idle":"2023-05-26T09:01:34.846207Z","shell.execute_reply.started":"2023-05-26T09:01:33.656530Z","shell.execute_reply":"2023-05-26T09:01:34.844738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Building models**","metadata":{}},{"cell_type":"code","source":"# Try three models with different architecture of layers\ninput_shape = [X_train.shape[1]]\n\nmodel_1 = Sequential([\n    layers.BatchNormalization(input_shape=input_shape),\n    Dense(units = 350, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 165, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 64, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 10, activation = 'linear')\n])\n\nmodel_2 = Sequential([\n    layers.BatchNormalization(input_shape=input_shape),\n    Dense(units = 80, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    Dense(units = 10, activation = 'linear')\n])\n\nmodel_3 = Sequential([\n    layers.BatchNormalization(input_shape=input_shape),\n    Dense(units = 560, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 700, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 430, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 120, activation = 'relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    \n    Dense(units = 10, activation = 'linear')\n])","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:03:43.770344Z","iopub.execute_input":"2023-05-26T09:03:43.770985Z","iopub.status.idle":"2023-05-26T09:03:44.653587Z","shell.execute_reply.started":"2023-05-26T09:03:43.770932Z","shell.execute_reply":"2023-05-26T09:03:44.652130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate each model \n\nmodels = [model_1, model_2, model_3]\nnn_train_loss = []\nnn_cv_loss    = []\n\nearly_stopping = keras.callbacks.EarlyStopping(\n    patience=5,\n    min_delta=0.001,\n    restore_best_weights=True,\n)\n\nfor model in models:\n    model.compile(\n        loss = SparseCategoricalCrossentropy(from_logits= True),\n        optimizer = tf.keras.optimizers.Adam(learning_rate = 1e-3),\n        metrics = ['accuracy']\n    )\n    \n    print(f\"Training model_{models.index(model) + 1}...\")\n    history = model.fit(\n        X_train, y_train, epochs = 50,\n        validation_data = (X_val, y_val),\n        batch_size = 50,\n        callbacks=[early_stopping],\n        verbose = 0\n    )\n    \n    history_df = pd.DataFrame(history.history)\n    history_df.loc[:, ['loss', 'val_loss']].plot(title=\"Cross-entropy\")\n    #history_df.loc[:, ['binary_accuracy', 'val_binary_accuracy']].plot(title=\"Accuracy\")\n    \n    print(\"Done!\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:05:12.146519Z","iopub.execute_input":"2023-05-26T09:05:12.146980Z","iopub.status.idle":"2023-05-26T09:09:15.850714Z","shell.execute_reply.started":"2023-05-26T09:05:12.146938Z","shell.execute_reply":"2023-05-26T09:09:15.849439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# It's seems like model_1 performs better\nbest_model = model_1\n\n# train on all the dataset\nbest_model.fit(X, y, callbacks=[early_stopping], batch_size = 50, epochs = 50)\n\n#predict\ntest_predictions = tf.nn.softmax(best_model.predict(X_test)).numpy()","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:12:36.929721Z","iopub.execute_input":"2023-05-26T09:12:36.930327Z","iopub.status.idle":"2023-05-26T09:18:45.785159Z","shell.execute_reply.started":"2023-05-26T09:12:36.930265Z","shell.execute_reply":"2023-05-26T09:18:45.783563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df.loc[:, ['loss', 'val_loss']].plot(title=\"Cross-entropy\")\nhistory_df.loc[:, ['accuracy', 'val_accuracy']].plot(title=\"Accuracy\")","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:20:57.125464Z","iopub.execute_input":"2023-05-26T09:20:57.126126Z","iopub.status.idle":"2023-05-26T09:20:57.793212Z","shell.execute_reply.started":"2023-05-26T09:20:57.126056Z","shell.execute_reply":"2023-05-26T09:20:57.791564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ImageId = []\nLabel = []\nfor i in range(len(test_predictions)):\n    ImageId.append(i+1)\n    Label.append(test_predictions[i].argmax())\n    \nsubmissions=pd.DataFrame({\n    \"ImageId\": ImageId,\n    \"Label\": Label\n})\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-26T09:23:55.723646Z","iopub.execute_input":"2023-05-26T09:23:55.724932Z","iopub.status.idle":"2023-05-26T09:23:55.884374Z","shell.execute_reply.started":"2023-05-26T09:23:55.724856Z","shell.execute_reply":"2023-05-26T09:23:55.882736Z"},"trusted":true},"execution_count":null,"outputs":[]}]}