{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Welcome to my first notebook!\nIn this notebook I will train an atoencoder NN and use it to train a SVM classifier. Although a SVM with radial basis function kernel works quite well with raw data, reducing the dimensionality will allow us to create a faster, lighter model and slightly improve its accuracy. Furthermore, the latent space could be exploited to easily train other classifiers.\n","metadata":{}},{"cell_type":"markdown","source":"# Importing libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\nfrom sklearn import metrics\nfrom sklearn.metrics import confusion_matrix\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import to_categorical\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading and scaling data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest  = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nX = train.drop(labels = [\"label\"],axis = 1)\ny = train['label']\nX = X / 255.0\ntest = test / 255.0\n\nX_train, X_val, Y_train, Y_val = train_test_split(X, y, test_size = 0.2, random_state=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:44:13.883900Z","iopub.execute_input":"2024-03-09T07:44:13.884606Z","iopub.status.idle":"2024-03-09T07:44:18.707964Z","shell.execute_reply.started":"2024-03-09T07:44:13.884568Z","shell.execute_reply":"2024-03-09T07:44:18.707055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Since the images are 28x28 pixels, we transform the 1D vectors to image format to train our NN.**","metadata":{}},{"cell_type":"code","source":"X_train_img = X_train.values.reshape(-1,28,28,1)\nX_val_img = X_val.values.reshape(-1,28,28,1)\ntest_img = test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:44:18.709267Z","iopub.execute_input":"2024-03-09T07:44:18.709634Z","iopub.status.idle":"2024-03-09T07:44:18.716491Z","shell.execute_reply.started":"2024-03-09T07:44:18.709588Z","shell.execute_reply":"2024-03-09T07:44:18.715616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Autoencoder","metadata":{}},{"cell_type":"markdown","source":"**We use keras to define an autoencoder from a sequence of convolutional layers**","metadata":{}},{"cell_type":"code","source":"input_shape = (28, 28, 1) \nlatent_dim = 32\nbatch_size = 32\nsteps_per_epoch = len(X_train)//batch_size\nvalidation_steps = len(X_val)//batch_size\nkeras.utils.set_random_seed(1)\n\n# Define the encoder layers\nencoder_layers = [\n    layers.InputLayer(shape=[28, 28, 1]),\n    layers.Conv2D(32, 3, activation='relu', padding='same'),\n    layers.MaxPooling2D(pool_size=(2, 2)),\n    layers.Conv2D(64, 3, activation='relu', padding='same'),\n    layers.MaxPooling2D(pool_size=(2, 2)),\n    layers.Flatten(),\n    layers.Dense(latent_dim, activation='relu')\n]\n\n# Define the decoder layers\ndecoder_layers = [\n    layers.InputLayer(shape=[latent_dim, ]),\n    layers.Dense(7 * 7 * 64, activation='relu'),\n    layers.Reshape((7, 7, 64)),\n    layers.Conv2DTranspose(64, 3, strides=2, activation='relu', padding='same'),\n    layers.Conv2DTranspose(32, 3, strides=2, activation='relu', padding='same'),\n    layers.Conv2D(1, 3, activation='sigmoid', padding='same')\n]\n\n# Create the encoder and decoder models\nencoder = Sequential(encoder_layers, name='encoder')\ndecoder = Sequential(decoder_layers, name='decoder')\n\n# Combine the encoder and decoder to create the autoencoder\nautoencoder = Sequential([encoder, decoder])\n\n# Compile the autoencoder\nautoencoder.compile(optimizer='adam', loss='binary_crossentropy')\n\n# Print summary of the autoencoder architecture\nencoder.summary()\ndecoder.summary()\nautoencoder.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:44:18.718828Z","iopub.execute_input":"2024-03-09T07:44:18.719099Z","iopub.status.idle":"2024-03-09T07:44:18.855663Z","shell.execute_reply.started":"2024-03-09T07:44:18.719076Z","shell.execute_reply":"2024-03-09T07:44:18.854699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We use the training images set to train the autoencoder. Some warnings appear, they don't seem to be a problem, but any comments on this would be welcome.**","metadata":{}},{"cell_type":"code","source":"hist = autoencoder.fit(X_train_img, X_train_img, batch_size = batch_size,\n                          validation_data = (X_val_img, X_val_img), epochs=25)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:44:18.856958Z","iopub.execute_input":"2024-03-09T07:44:18.857325Z","iopub.status.idle":"2024-03-09T07:45:55.833291Z","shell.execute_reply.started":"2024-03-09T07:44:18.857291Z","shell.execute_reply":"2024-03-09T07:45:55.832377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_frame = pd.DataFrame(hist.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:45:55.834840Z","iopub.execute_input":"2024-03-09T07:45:55.835120Z","iopub.status.idle":"2024-03-09T07:45:56.075333Z","shell.execute_reply.started":"2024-03-09T07:45:55.835096Z","shell.execute_reply":"2024-03-09T07:45:56.074398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can use our trained autoencoder to represent the images as 32 values**","metadata":{}},{"cell_type":"code","source":"encoder_model = autoencoder.layers[0]\nencoded_Train = encoder_model.predict(X_train_img)\nencoded_test = encoder_model.predict(test_img)\ndecoder_model = autoencoder.layers[1]","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:45:56.076761Z","iopub.execute_input":"2024-03-09T07:45:56.077194Z","iopub.status.idle":"2024-03-09T07:46:00.247939Z","shell.execute_reply.started":"2024-03-09T07:45:56.077157Z","shell.execute_reply":"2024-03-09T07:46:00.247062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_val = encoder_model.predict(X_val_img)\nreconstructed_val = decoder_model.predict(encoded_val)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:46:00.249088Z","iopub.execute_input":"2024-03-09T07:46:00.249398Z","iopub.status.idle":"2024-03-09T07:46:01.842086Z","shell.execute_reply.started":"2024-03-09T07:46:00.249372Z","shell.execute_reply":"2024-03-09T07:46:01.841040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can verify that the decoder reconstructs the images quite well from their representation, so not much relevant information is lost**","metadata":{}},{"cell_type":"code","source":"X_val_img_first_4 = X_val_img[:4]\nreconstructed_val_first_4 = reconstructed_val[:4]\n\nfig, axes = plt.subplots(nrows=2, ncols=4, figsize=(12, 6))\n\nfor i, img in enumerate(X_val_img_first_4):\n    ax = axes[0, i]\n    ax.imshow(img, cmap='gray')  \n    ax.set_title('Original {}'.format(i+1))\n    ax.axis('off')\n\nfor i, img in enumerate(reconstructed_val_first_4):\n    ax = axes[1, i]\n    ax.imshow(img, cmap='gray')  \n    ax.set_title('Reconstruida {}'.format(i+1))\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:46:01.843979Z","iopub.execute_input":"2024-03-09T07:46:01.844295Z","iopub.status.idle":"2024-03-09T07:46:02.911315Z","shell.execute_reply.started":"2024-03-09T07:46:01.844269Z","shell.execute_reply":"2024-03-09T07:46:02.910066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SVM","metadata":{}},{"cell_type":"markdown","source":"**Finally, let's train the SVM**","metadata":{}},{"cell_type":"code","source":"model = SVC(kernel='rbf', random_state=1)\nmodel.fit(encoded_Train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:46:02.915367Z","iopub.execute_input":"2024-03-09T07:46:02.915945Z","iopub.status.idle":"2024-03-09T07:46:09.422665Z","shell.execute_reply.started":"2024-03-09T07:46:02.915903Z","shell.execute_reply":"2024-03-09T07:46:09.421519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The model trains quickly and obtains an accuracy of more than 97%**","metadata":{}},{"cell_type":"code","source":"predictions = model.predict(encoded_val)\nprint(\"accuracy:\", metrics.accuracy_score(y_true=Y_val, y_pred=predictions), \"\\n\")\nprint(metrics.confusion_matrix(y_true=Y_val, y_pred=predictions))","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:46:09.423910Z","iopub.execute_input":"2024-03-09T07:46:09.424220Z","iopub.status.idle":"2024-03-09T07:46:12.564671Z","shell.execute_reply.started":"2024-03-09T07:46:09.424195Z","shell.execute_reply":"2024-03-09T07:46:12.563636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submission:**","metadata":{}},{"cell_type":"code","source":"predictions = model.predict(encoded_test)\n\ntest_pred_df=pd.DataFrame({\"ImageId\":range(1,len(test)+1),\"label\":predictions})\ntest_pred_df.to_csv(\"submission.csv\",index=False)\nprint(test_pred_df)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:46:12.566074Z","iopub.execute_input":"2024-03-09T07:46:12.566446Z","iopub.status.idle":"2024-03-09T07:46:23.222057Z","shell.execute_reply.started":"2024-03-09T07:46:12.566412Z","shell.execute_reply":"2024-03-09T07:46:23.221075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thank you so much!**","metadata":{}}]}