{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport sklearn as sk","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:49.832908Z","iopub.execute_input":"2024-07-21T11:33:49.833509Z","iopub.status.idle":"2024-07-21T11:33:49.837632Z","shell.execute_reply.started":"2024-07-21T11:33:49.833464Z","shell.execute_reply":"2024-07-21T11:33:49.836586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:49.883890Z","iopub.execute_input":"2024-07-21T11:33:49.884393Z","iopub.status.idle":"2024-07-21T11:33:52.647925Z","shell.execute_reply.started":"2024-07-21T11:33:49.884365Z","shell.execute_reply":"2024-07-21T11:33:52.647051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df[\"label\"].values\ndf = df.drop(\"label\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:52.649917Z","iopub.execute_input":"2024-07-21T11:33:52.650275Z","iopub.status.idle":"2024-07-21T11:33:52.732684Z","shell.execute_reply.started":"2024-07-21T11:33:52.650242Z","shell.execute_reply":"2024-07-21T11:33:52.731558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.asarray(df).shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:52.734021Z","iopub.execute_input":"2024-07-21T11:33:52.734341Z","iopub.status.idle":"2024-07-21T11:33:52.740944Z","shell.execute_reply.started":"2024-07-21T11:33:52.734314Z","shell.execute_reply":"2024-07-21T11:33:52.739897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = np.asarray(df).reshape(42000, 28, 28)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:52.743463Z","iopub.execute_input":"2024-07-21T11:33:52.744086Z","iopub.status.idle":"2024-07-21T11:33:52.749447Z","shell.execute_reply.started":"2024-07-21T11:33:52.744052Z","shell.execute_reply":"2024-07-21T11:33:52.748642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:52.750551Z","iopub.execute_input":"2024-07-21T11:33:52.750847Z","iopub.status.idle":"2024-07-21T11:33:52.758938Z","shell.execute_reply.started":"2024-07-21T11:33:52.750824Z","shell.execute_reply":"2024-07-21T11:33:52.757981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:52.760228Z","iopub.execute_input":"2024-07-21T11:33:52.760706Z","iopub.status.idle":"2024-07-21T11:33:52.768637Z","shell.execute_reply.started":"2024-07-21T11:33:52.760657Z","shell.execute_reply":"2024-07-21T11:33:52.767711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data visualization","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfigs, axs = plt.subplots(5,5, figsize=(10, 10))\naxs = axs.flatten()\nfor ax, i in zip(axs, range(25)):\n    ax.imshow(x[i], cmap=\"gray\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:52.769842Z","iopub.execute_input":"2024-07-21T11:33:52.770179Z","iopub.status.idle":"2024-07-21T11:33:55.297832Z","shell.execute_reply.started":"2024-07-21T11:33:52.770149Z","shell.execute_reply":"2024-07-21T11:33:55.296898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{}},{"cell_type":"code","source":"from sklearn import model_selection\nx_train, x_val, y_train, y_val = sk.model_selection.train_test_split(x, y, test_size=0.3)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.299229Z","iopub.execute_input":"2024-07-21T11:33:55.299687Z","iopub.status.idle":"2024-07-21T11:33:55.677304Z","shell.execute_reply.started":"2024-07-21T11:33:55.299654Z","shell.execute_reply":"2024-07-21T11:33:55.676474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x\ndel y","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.678389Z","iopub.execute_input":"2024-07-21T11:33:55.678698Z","iopub.status.idle":"2024-07-21T11:33:55.683550Z","shell.execute_reply.started":"2024-07-21T11:33:55.678673Z","shell.execute_reply":"2024-07-21T11:33:55.682692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will scale our features to be in the range [0,1].\nx_train_scaled = x_train / 255.0\nx_val_scaled = x_val / 255.0\n\n# Since our labels are categories, we should use OneHotEncoder.\nfrom sklearn import preprocessing\n\nohe = sk.preprocessing.OneHotEncoder()\ny_train_encoded = ohe.fit_transform(np.asarray(y_train).reshape(-1, 1)).todense()\ny_val_encoded = ohe.transform(np.asarray(y_val).reshape(-1, 1)).todense()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.687369Z","iopub.execute_input":"2024-07-21T11:33:55.688068Z","iopub.status.idle":"2024-07-21T11:33:55.797027Z","shell.execute_reply.started":"2024-07-21T11:33:55.688034Z","shell.execute_reply":"2024-07-21T11:33:55.795941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train\ndel x_val\ndel y_train\ndel y_val","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.798268Z","iopub.execute_input":"2024-07-21T11:33:55.798615Z","iopub.status.idle":"2024-07-21T11:33:55.804119Z","shell.execute_reply.started":"2024-07-21T11:33:55.798588Z","shell.execute_reply":"2024-07-21T11:33:55.803216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_scaled.shape, x_val_scaled.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.805483Z","iopub.execute_input":"2024-07-21T11:33:55.805899Z","iopub.status.idle":"2024-07-21T11:33:55.815287Z","shell.execute_reply.started":"2024-07-21T11:33:55.805863Z","shell.execute_reply":"2024-07-21T11:33:55.814299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_encoded.shape, y_val_encoded.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.816540Z","iopub.execute_input":"2024-07-21T11:33:55.816843Z","iopub.status.idle":"2024-07-21T11:33:55.825435Z","shell.execute_reply.started":"2024-07-21T11:33:55.816819Z","shell.execute_reply":"2024-07-21T11:33:55.824552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the model","metadata":{}},{"cell_type":"code","source":"def build_model():\n    inputs = tf.keras.Input(shape=(28,28,1))\n    x = tf.keras.layers.Conv2D(32, kernel_size=(3,3), activation=\"relu\")(inputs)\n    x = tf.keras.layers.AveragePooling2D(pool_size=(2,2))(x)\n    x = tf.keras.layers.Conv2D(64, kernel_size=(3,3), activation=\"relu\")(x)\n    x = tf.keras.layers.AveragePooling2D(pool_size=(2,2))(x)\n    x = tf.keras.layers.Flatten()(x)\n    x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    outputs = tf.keras.layers.Dense(10, activation=\"softmax\")(x)\n    \n    model = tf.keras.Model(inputs, outputs)\n    return model\n\nmodel = build_model()\nmodel.summary(show_trainable=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.826574Z","iopub.execute_input":"2024-07-21T11:33:55.827098Z","iopub.status.idle":"2024-07-21T11:33:55.892634Z","shell.execute_reply.started":"2024-07-21T11:33:55.827074Z","shell.execute_reply":"2024-07-21T11:33:55.891830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"init_learning_rate = 1e-3\nepochs = 30\nscheduler = tf.keras.optimizers.schedules.CosineDecay(init_learning_rate, 1000, alpha=1e-3)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:33:55.893574Z","iopub.execute_input":"2024-07-21T11:33:55.893829Z","iopub.status.idle":"2024-07-21T11:33:55.898313Z","shell.execute_reply.started":"2024-07-21T11:33:55.893806Z","shell.execute_reply":"2024-07-21T11:33:55.897426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.SGD(scheduler),\n              loss=tf.keras.losses.CategoricalCrossentropy(),\n              metrics=[\"categorical_accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:40:43.470383Z","iopub.execute_input":"2024-07-21T11:40:43.471174Z","iopub.status.idle":"2024-07-21T11:40:43.479950Z","shell.execute_reply.started":"2024-07-21T11:40:43.471142Z","shell.execute_reply":"2024-07-21T11:40:43.479142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x=x_train_scaled, y=y_train_encoded, validation_data=(x_val_scaled, y_val_encoded), epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:40:43.976714Z","iopub.execute_input":"2024-07-21T11:40:43.977102Z","iopub.status.idle":"2024-07-21T11:41:57.899666Z","shell.execute_reply.started":"2024-07-21T11:40:43.977072Z","shell.execute_reply":"2024-07-21T11:41:57.898825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Error analysis","metadata":{}},{"cell_type":"code","source":"figs, axs = plt.subplots(2,1, figsize=(15,12))\naxs[0].plot(range(epochs), history.history[\"categorical_accuracy\"])\naxs[0].set_title(\"Training categorical accuracy\")\naxs[0].set_xlabel(\"Epochs\")\naxs[0].set_ylabel(\"Accuracy\")\naxs[1].plot(range(epochs), history.history[\"val_categorical_accuracy\"])\naxs[1].set_title(\"Validation categorical accuracy\")\naxs[1].set_xlabel(\"Epochs\")\naxs[1].set_ylabel(\"Accuracy\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:46:05.471526Z","iopub.execute_input":"2024-07-21T11:46:05.471894Z","iopub.status.idle":"2024-07-21T11:46:06.002379Z","shell.execute_reply.started":"2024-07-21T11:46:05.471865Z","shell.execute_reply":"2024-07-21T11:46:06.001468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction on the test set","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:47:11.319151Z","iopub.execute_input":"2024-07-21T11:47:11.319880Z","iopub.status.idle":"2024-07-21T11:47:13.425153Z","shell.execute_reply.started":"2024-07-21T11:47:11.319846Z","shell.execute_reply":"2024-07-21T11:47:13.424241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.asarray(test).shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:48:04.824304Z","iopub.execute_input":"2024-07-21T11:48:04.824634Z","iopub.status.idle":"2024-07-21T11:48:04.830728Z","shell.execute_reply.started":"2024-07-21T11:48:04.824605Z","shell.execute_reply":"2024-07-21T11:48:04.829786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = np.asarray(test).reshape(28000, 28, 28)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:48:40.768355Z","iopub.execute_input":"2024-07-21T11:48:40.769306Z","iopub.status.idle":"2024-07-21T11:48:40.776760Z","shell.execute_reply.started":"2024-07-21T11:48:40.769265Z","shell.execute_reply":"2024-07-21T11:48:40.775693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yhat_encoded = model.predict(test)\nyhat_encoded.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:49:58.203522Z","iopub.execute_input":"2024-07-21T11:49:58.204355Z","iopub.status.idle":"2024-07-21T11:50:00.154726Z","shell.execute_reply.started":"2024-07-21T11:49:58.204323Z","shell.execute_reply":"2024-07-21T11:50:00.153685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yhat = ohe.inverse_transform(yhat_encoded)\nyhat.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:50:07.116717Z","iopub.execute_input":"2024-07-21T11:50:07.117460Z","iopub.status.idle":"2024-07-21T11:50:07.125040Z","shell.execute_reply.started":"2024-07-21T11:50:07.117409Z","shell.execute_reply":"2024-07-21T11:50:07.124220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yhat","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:50:13.403228Z","iopub.execute_input":"2024-07-21T11:50:13.403617Z","iopub.status.idle":"2024-07-21T11:50:13.410180Z","shell.execute_reply.started":"2024-07-21T11:50:13.403587Z","shell.execute_reply":"2024-07-21T11:50:13.409175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transferring results to a csv file","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({\"ImageId\": range(1, yhat.shape[0]+1),\n                 \"Label\": yhat.reshape(-1)})\ndf.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T11:58:48.496100Z","iopub.execute_input":"2024-07-21T11:58:48.496465Z","iopub.status.idle":"2024-07-21T11:58:48.534884Z","shell.execute_reply.started":"2024-07-21T11:58:48.496436Z","shell.execute_reply":"2024-07-21T11:58:48.534134Z"},"trusted":true},"execution_count":null,"outputs":[]}]}