{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <center>Training 2D CNN Keras model to classify digits using gradient tape method</center>\n","metadata":{}},{"cell_type":"markdown","source":"## Introduction\nIn this experiment we use Tensorflows `GradientTape()` method to train a CNN manually by logging the prediction error, compute loss and backpropagate the computed gradients throght the CNN. To ensure that this method achives comparable performance with the `model.fit()` method we do 10-times cross-validation and compare the training curves (mean $\\pm$ standard deviation) when using the `GradientTape()` method versus the `model.fit()` method.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport os\nimport math\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nimport seaborn as sns\nimport time","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-07T19:07:24.230208Z","iopub.execute_input":"2023-02-07T19:07:24.231480Z","iopub.status.idle":"2023-02-07T19:07:29.817982Z","shell.execute_reply.started":"2023-02-07T19:07:24.231337Z","shell.execute_reply":"2023-02-07T19:07:29.817015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data description (train and test)","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nprint(df_train.describe())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-07T19:07:29.819744Z","iopub.execute_input":"2023-02-07T19:07:29.820408Z","iopub.status.idle":"2023-02-07T19:07:35.054859Z","shell.execute_reply.started":"2023-02-07T19:07:29.820369Z","shell.execute_reply":"2023-02-07T19:07:35.053037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nprint(df_test.describe())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-07T19:07:35.056365Z","iopub.execute_input":"2023-02-07T19:07:35.056740Z","iopub.status.idle":"2023-02-07T19:07:38.862902Z","shell.execute_reply.started":"2023-02-07T19:07:35.056705Z","shell.execute_reply":"2023-02-07T19:07:38.861463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing","metadata":{}},{"cell_type":"code","source":"# Separate digit data and labels\nX_dev = df_train.iloc[:,1:]\ny_dev=df_train.iloc[:,0]\n\n# Reshape digit data to images\nX_dev = X_dev.values.reshape(-1,int(math.sqrt(X_dev.shape[1])),int(math.sqrt(X_dev.shape[1])),1)\n\n#one-hot encode labels (0-9)\ny_dev = to_categorical(y_dev)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:38.866525Z","iopub.execute_input":"2023-02-07T19:07:38.867607Z","iopub.status.idle":"2023-02-07T19:07:38.876835Z","shell.execute_reply.started":"2023-02-07T19:07:38.867567Z","shell.execute_reply":"2023-02-07T19:07:38.874925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.bar(x = np.unique(np.argmax(y_dev,axis=1)),height = np.unique(np.argmax(y_dev,axis=1), return_counts=True)[1])\nplt.xticks([0,1,2,3,4,5,6,7,8,9])\nplt.ylabel(\"Amount\")\nplt.xlabel(\"Digits\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:38.878569Z","iopub.execute_input":"2023-02-07T19:07:38.879558Z","iopub.status.idle":"2023-02-07T19:07:39.176318Z","shell.execute_reply.started":"2023-02-07T19:07:38.879511Z","shell.execute_reply":"2023-02-07T19:07:39.175473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(36,3))\nfor i in range(10):\n    plt.subplot(1,10,i+1)\n    plt.imshow(X_dev[i])\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-07T19:07:39.178221Z","iopub.execute_input":"2023-02-07T19:07:39.178822Z","iopub.status.idle":"2023-02-07T19:07:40.177269Z","shell.execute_reply.started":"2023-02-07T19:07:39.178784Z","shell.execute_reply":"2023-02-07T19:07:40.176350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cnn_model():\n    inputlayer = tf.keras.layers.Input((28,28,1))\n    conv1 = tf.keras.layers.Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu')(inputlayer)\n    conv2 = tf.keras.layers.Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu')(conv1)\n    mp1 = tf.keras.layers.MaxPool2D(pool_size=(2,2))(conv2)\n    drop1 = tf.keras.layers.Dropout(0.25)(mp1)\n    conv3 = tf.keras.layers.Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu')(drop1)\n    conv4 = tf.keras.layers.Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', activation ='relu')(conv3)\n    mp2 = tf.keras.layers.MaxPool2D(pool_size=(2,2))(conv4)\n    drop2 = tf.keras.layers.Dropout(0.25)(mp2)\n    flatten = tf.keras.layers.Flatten()(drop2)\n    dense1 = tf.keras.layers.Dense(256, activation = \"relu\")(flatten)\n    drop3 = tf.keras.layers.Dropout(0.25)(dense1)\n    out = tf.keras.layers.Dense(10, activation = \"softmax\")(drop3)\n    model = tf.keras.models.Model(inputs=inputlayer, outputs=out)\n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-07T19:07:40.178978Z","iopub.execute_input":"2023-02-07T19:07:40.179344Z","iopub.status.idle":"2023-02-07T19:07:40.191673Z","shell.execute_reply.started":"2023-02-07T19:07:40.179306Z","shell.execute_reply":"2023-02-07T19:07:40.190695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model architecture","metadata":{}},{"cell_type":"code","source":"test_model = cnn_model()\nprint(test_model.summary())","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:40.192918Z","iopub.execute_input":"2023-02-07T19:07:40.193873Z","iopub.status.idle":"2023-02-07T19:07:43.315330Z","shell.execute_reply.started":"2023-02-07T19:07:40.193838Z","shell.execute_reply":"2023-02-07T19:07:43.314389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntf.keras.utils.plot_model(test_model,to_file='model.png')\n","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:43.316793Z","iopub.execute_input":"2023-02-07T19:07:43.317343Z","iopub.status.idle":"2023-02-07T19:07:44.693578Z","shell.execute_reply.started":"2023-02-07T19:07:43.317306Z","shell.execute_reply":"2023-02-07T19:07:44.690026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining number of cross-validation splits","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=10)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:44.697896Z","iopub.execute_input":"2023-02-07T19:07:44.698327Z","iopub.status.idle":"2023-02-07T19:07:44.707811Z","shell.execute_reply.started":"2023-02-07T19:07:44.698284Z","shell.execute_reply":"2023-02-07T19:07:44.706604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hyperparameters","metadata":{}},{"cell_type":"code","source":"training_epochs = 100\nbatch_size = 64\nsteps = int((len(X_dev)*(9/10))//batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:44.709786Z","iopub.execute_input":"2023-02-07T19:07:44.710318Z","iopub.status.idle":"2023-02-07T19:07:44.718624Z","shell.execute_reply.started":"2023-02-07T19:07:44.710286Z","shell.execute_reply":"2023-02-07T19:07:44.717462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit models","metadata":{}},{"cell_type":"markdown","source":"### Using `model.fit()`","metadata":{}},{"cell_type":"code","source":"df_fit = pd.DataFrame({\"epoch\":[], \"loss\": [],\"accuracy\": [] })\n\nfor i, (train_index, test_index) in enumerate(skf.split(X_dev,np.argmax(y_dev,axis=1))):\n    print(\"---------------\")\n    print(\"Fold \",i+1)\n    print(\"---------------\")\n    X_train, X_test = X_dev[train_index], X_dev[test_index]\n    y_train, y_test = y_dev[train_index], y_dev[test_index]\n    model = cnn_model()\n    model.compile(optimizer = \"adam\" , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])\n    history = model.fit(x=X_train,y=y_train,epochs=training_epochs, validation_data=(X_test,y_test),\n                        steps_per_epoch=steps,batch_size=batch_size)\n    for j in range(training_epochs):\n        df_fit = df_fit.append({\"epoch\":j+1,\"loss\":history.history[\"val_loss\"][j],\"accuracy\": history.history[\"val_accuracy\"][j]},ignore_index=True)\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-07T19:08:26.111616Z","iopub.execute_input":"2023-02-07T19:08:26.112116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_step(X, y):\n    with tf.GradientTape() as tape:\n        pred = model(X)\n        loss = tf.keras.losses.categorical_crossentropy(y, pred)\n\n    grads = tape.gradient(loss, model.trainable_variables)\n    opt.apply_gradients(zip(grads, model.trainable_variables))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-07T19:07:45.097172Z","iopub.status.idle":"2023-02-07T19:07:45.097957Z","shell.execute_reply.started":"2023-02-07T19:07:45.097699Z","shell.execute_reply":"2023-02-07T19:07:45.097723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Using `tf.GradientTape()`","metadata":{}},{"cell_type":"code","source":"df_gradtape = pd.DataFrame({\"epoch\":[], \"loss\": [],\"accuracy\": [] })\nfor i, (train_index, test_index) in enumerate(skf.split(X_dev,np.argmax(y_dev,axis=1))):\n    print(\"---------------\")\n    print(\"Fold \",i+1)\n    print(\"---------------\")\n    X_train, X_test = X_dev[train_index], X_dev[test_index]\n    y_train, y_test = y_dev[train_index], y_dev[test_index]\n    model = cnn_model()\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    start_epoch = time.time()\n    for epoch in range(training_epochs):\n        print(\"epoch {}\".format(epoch+1))\n        for step in range(steps):\n            start_idx=batch_size*step\n            end_idx=batch_size*(step+1)\n            X_batch = X_train[start_idx:end_idx]\n            y_batch = y_train[start_idx:end_idx]\n            training_step(X_batch,y_batch)\n        print(\"Epoch training time: {} seconds\".format(round(time.time()-start_epoch,1)))\n        y_test_hat = model.predict(X_test)\n        avg_loss = np.mean(tf.keras.losses.categorical_crossentropy(y_test, y_test_hat))\n        acc = accuracy_score(np.argmax(y_test,axis=1),np.argmax(model.predict(X_test),axis=1))\n        df_gradtape = df_gradtape.append({\"epoch\":epoch+1,\"loss\":avg_loss,\"accuracy\": acc},ignore_index=True)\n        print(\"Loss = {}, Accuracy = {} \".format(round(avg_loss,3),round(acc,3)))\n    print(\"---------------\")","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-07T19:07:45.099460Z","iopub.status.idle":"2023-02-07T19:07:45.100189Z","shell.execute_reply.started":"2023-02-07T19:07:45.099946Z","shell.execute_reply":"2023-02-07T19:07:45.099970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore difference between `model.fit()` and `tf.GradientTape`","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.title(\"Comparing loss curves using model.fit() and tf.GradientTape()\", fontsize=16)\nsns.lineplot(x=df_fit[\"epoch\"], y=df_fit[\"loss\"])\nsns.lineplot(x=df_gradtape[\"epoch\"], y=df_gradtape[\"loss\"])\nplt.legend([\"model.fit() (mean)\",\"model.fit() (std)\",\"tf.GradientTape() (mean)\",\"tf.GradientTape() (std)\"],loc='upper right')\nplt.savefig(\"loss_gt_mf.png\",dpi=100)\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:45.101547Z","iopub.status.idle":"2023-02-07T19:07:45.102266Z","shell.execute_reply.started":"2023-02-07T19:07:45.102020Z","shell.execute_reply":"2023-02-07T19:07:45.102043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.title(\"Comparing accuracy curves using model.fit() and tf.GradientTape()\", fontsize=16)\nsns.lineplot(x=df_fit[\"epoch\"], y=df_fit[\"accuracy\"])\nsns.lineplot(x=df_gradtape[\"epoch\"], y=df_gradtape[\"accuracy\"])\nplt.legend([\"model.fit() (mean)\",\"model.fit() (std)\",\"tf.GradientTape() (mean)\",\"tf.GradientTape() (std)\"],loc='lower right')\nplt.savefig(\"accuracy_gt_mf.png\",dpi=100)\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:07:45.103761Z","iopub.status.idle":"2023-02-07T19:07:45.104637Z","shell.execute_reply.started":"2023-02-07T19:07:45.104321Z","shell.execute_reply":"2023-02-07T19:07:45.104346Z"},"trusted":true},"execution_count":null,"outputs":[]}]}