{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:30:31.672780Z","iopub.execute_input":"2025-03-31T10:30:31.673048Z","iopub.status.idle":"2025-03-31T10:30:32.978210Z","shell.execute_reply.started":"2025-03-31T10:30:31.673021Z","shell.execute_reply":"2025-03-31T10:30:32.976929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:18:38.185127Z","iopub.execute_input":"2025-03-31T12:18:38.185981Z","iopub.status.idle":"2025-03-31T12:18:40.892015Z","shell.execute_reply.started":"2025-03-31T12:18:38.185937Z","shell.execute_reply":"2025-03-31T12:18:40.890890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:50:07.557599Z","iopub.execute_input":"2025-03-31T10:50:07.558042Z","iopub.status.idle":"2025-03-31T10:50:07.614121Z","shell.execute_reply.started":"2025-03-31T10:50:07.558009Z","shell.execute_reply":"2025-03-31T10:50:07.613047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Slicing the dataset to separate feature matrix 'X' and the vector of predictions 'y'\n\nX = dataset.iloc[:, 1:].values\ny = dataset.iloc[:, 0].values\n\nX.shape, y.shape, X.dtype, y.dtype","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:51:43.684071Z","iopub.execute_input":"2025-03-31T10:51:43.684389Z","iopub.status.idle":"2025-03-31T10:51:43.691411Z","shell.execute_reply.started":"2025-03-31T10:51:43.684365Z","shell.execute_reply":"2025-03-31T10:51:43.690130Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nrandom_indexes = np.random.choice(range(len(X)), 25)\nprint(\"Random Indexes : \", random_indexes)\n\nX_random = X[random_indexes]\ny_random = y[random_indexes]\n\nprint(\"\\n Shape : \", X_random.shape)\n\nplt.figure(figsize = (16, 10))\nfor i in range(25):\n    image = X_random[i]\n    image = image.reshape((28, 28))\n    plt.subplot(5, 5, i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(image, \"binary\")\n    plt.title('label : {}'.format(y_random[i]))\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:52:23.221916Z","iopub.execute_input":"2025-03-31T10:52:23.222267Z","iopub.status.idle":"2025-03-31T10:52:24.644914Z","shell.execute_reply.started":"2025-03-31T10:52:23.222240Z","shell.execute_reply":"2025-03-31T10:52:24.643559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualizing the dataset by reshaping an image into the original format i.e. 28 * 28\n\nfirst_image = X[1]\nfirst_image = first_image.reshape((28, 28))\n\nplt.imshow(first_image)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:53:53.694232Z","iopub.execute_input":"2025-03-31T10:53:53.694632Z","iopub.status.idle":"2025-03-31T10:53:53.851084Z","shell.execute_reply.started":"2025-03-31T10:53:53.694599Z","shell.execute_reply":"2025-03-31T10:53:53.849808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(first_image, \"binary\")\nplt.title('label : {}'.format(y[1]))\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:54:08.117846Z","iopub.execute_input":"2025-03-31T10:54:08.118198Z","iopub.status.idle":"2025-03-31T10:54:08.221844Z","shell.execute_reply.started":"2025-03-31T10:54:08.118171Z","shell.execute_reply":"2025-03-31T10:54:08.220721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying PCA to reduce dimensionality\n\nfrom sklearn.decomposition import PCA\npca = PCA()\npca.fit(X)\ncumsum = np.cumsum(pca.explained_variance_ratio_)\nd = np.argmax(cumsum >= 0.95) + 1\nd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:54:31.613595Z","iopub.execute_input":"2025-03-31T10:54:31.613979Z","iopub.status.idle":"2025-03-31T10:54:36.911020Z","shell.execute_reply.started":"2025-03-31T10:54:31.613946Z","shell.execute_reply":"2025-03-31T10:54:36.909783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca = PCA(n_components = 154)\nX_reduced = pca.fit_transform(X)\nX_reduced.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:54:49.190215Z","iopub.execute_input":"2025-03-31T10:54:49.190937Z","iopub.status.idle":"2025-03-31T10:54:52.211643Z","shell.execute_reply.started":"2025-03-31T10:54:49.190891Z","shell.execute_reply":"2025-03-31T10:54:52.210429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying T-SNE Algorithm to check arrangements in the dataset\n\nm = 5000\nidx = np.random.permutation(len(X))[:m]\n\nX_tsne = X[idx]\ny_tsne = y[idx]\n\nfrom sklearn.manifold import TSNE\ntsne = TSNE(n_components = 2)\nX_2d = tsne.fit_transform(X_tsne)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:56:02.573560Z","iopub.execute_input":"2025-03-31T10:56:02.573929Z","iopub.status.idle":"2025-03-31T10:56:28.846508Z","shell.execute_reply.started":"2025-03-31T10:56:02.573900Z","shell.execute_reply":"2025-03-31T10:56:28.845627Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (16, 10))\nplt.scatter(X_2d[:, 0], X_2d[:, 1], c = y_tsne, cmap = 'jet')\nplt.axis('off')\nplt.colorbar()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:56:34.950979Z","iopub.execute_input":"2025-03-31T10:56:34.951373Z","iopub.status.idle":"2025-03-31T10:56:35.312910Z","shell.execute_reply.started":"2025-03-31T10:56:34.951328Z","shell.execute_reply":"2025-03-31T10:56:35.311668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying feature scaling for faster convergence\n\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nX = scaler.fit_transform(X)\nX","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:56:49.230074Z","iopub.execute_input":"2025-03-31T10:56:49.230435Z","iopub.status.idle":"2025-03-31T10:56:49.858050Z","shell.execute_reply.started":"2025-03-31T10:56:49.230387Z","shell.execute_reply":"2025-03-31T10:56:49.856670Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Logistic Regression\n\nfrom sklearn.linear_model import LogisticRegression\nlog_reg = LogisticRegression(solver = 'saga', tol = 0.1, penalty = 'l1')\nlog_reg.fit(X, y)\n\n%time log_reg.score(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:56:55.813991Z","iopub.execute_input":"2025-03-31T10:56:55.814326Z","iopub.status.idle":"2025-03-31T10:57:14.831907Z","shell.execute_reply.started":"2025-03-31T10:56:55.814300Z","shell.execute_reply":"2025-03-31T10:57:14.830846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Naive Bayes Algorithm\n\nfrom sklearn.naive_bayes import GaussianNB\nnaive_bayes = GaussianNB()\nnaive_bayes.fit(X, y)\n\n%time naive_bayes.score(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:57:25.358227Z","iopub.execute_input":"2025-03-31T10:57:25.358633Z","iopub.status.idle":"2025-03-31T10:57:28.283508Z","shell.execute_reply.started":"2025-03-31T10:57:25.358600Z","shell.execute_reply":"2025-03-31T10:57:28.282306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Decision Tree Algorithm\n\nfrom sklearn.tree import DecisionTreeClassifier\ndtf = DecisionTreeClassifier()\ndtf.fit(X, y)\n\n%time dtf.score(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:57:38.030847Z","iopub.execute_input":"2025-03-31T10:57:38.031273Z","iopub.status.idle":"2025-03-31T10:57:51.582540Z","shell.execute_reply.started":"2025-03-31T10:57:38.031231Z","shell.execute_reply":"2025-03-31T10:57:51.581435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Using Cross Validation with Decision Tree to avoid overfitting\n\nfrom sklearn.model_selection import cross_val_score\ncross_val_score(dtf, X, y, cv = 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T10:59:45.833589Z","iopub.execute_input":"2025-03-31T10:59:45.834041Z","iopub.status.idle":"2025-03-31T11:00:38.370098Z","shell.execute_reply.started":"2025-03-31T10:59:45.834006Z","shell.execute_reply":"2025-03-31T11:00:38.369054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Ensemble techniques using a Random Forest Classifier\n\nfrom sklearn.ensemble import RandomForestClassifier\nrf = RandomForestClassifier(n_estimators = 20, max_depth = 20)\nrf.fit(X, y)\n\n%time rf.score(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:00:38.371611Z","iopub.execute_input":"2025-03-31T11:00:38.371912Z","iopub.status.idle":"2025-03-31T11:00:44.940999Z","shell.execute_reply.started":"2025-03-31T11:00:38.371885Z","shell.execute_reply":"2025-03-31T11:00:44.939983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Cross Validation on the Random Forest estimator to avoid overfitting\n\ncross_val_score(rf, X, y, cv = 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:00:44.943162Z","iopub.execute_input":"2025-03-31T11:00:44.943607Z","iopub.status.idle":"2025-03-31T11:01:08.263629Z","shell.execute_reply.started":"2025-03-31T11:00:44.943565Z","shell.execute_reply":"2025-03-31T11:01:08.262604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Ensemble techniques using a Random Forest Classifier on the Reduced Dataset\n\nfrom sklearn.ensemble import RandomForestClassifier\nrf1 = RandomForestClassifier(n_estimators = 20, max_depth = 20)\nrf1.fit(X_reduced, y)\n\n%time rf1.score(X_reduced, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:01:08.264986Z","iopub.execute_input":"2025-03-31T11:01:08.265325Z","iopub.status.idle":"2025-03-31T11:01:23.685294Z","shell.execute_reply.started":"2025-03-31T11:01:08.265297Z","shell.execute_reply":"2025-03-31T11:01:23.684320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Cross Validation on the Random Forest estimator to avoid overfitting\n\ncross_val_score(rf1, X_reduced, y, cv = 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:01:52.166295Z","iopub.execute_input":"2025-03-31T11:01:52.166696Z","iopub.status.idle":"2025-03-31T11:02:50.845244Z","shell.execute_reply.started":"2025-03-31T11:01:52.166664Z","shell.execute_reply":"2025-03-31T11:02:50.843998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generating predictions on the training set\n\nfrom sklearn.model_selection import cross_val_predict\ny_pred = cross_val_predict(rf, X, y, cv = 5)\ny_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:02:50.846639Z","iopub.execute_input":"2025-03-31T11:02:50.846971Z","iopub.status.idle":"2025-03-31T11:03:13.880216Z","shell.execute_reply.started":"2025-03-31T11:02:50.846945Z","shell.execute_reply":"2025-03-31T11:03:13.879119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Checking the Precision, Recall, F1 Score and plotting the confusion matrix\n\nfrom sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score\n\nprint('Accuracy : ', accuracy_score(y, y_pred))\nprint('\\nPrecision : ', precision_score(y, y_pred, average = \"micro\"))\nprint('\\nRecall : ', recall_score(y, y_pred, average = \"micro\"))\nprint('\\nF1 Score : ', f1_score(y, y_pred, average = \"micro\"))\nprint('\\nConfusion Matrix \\n\\n', confusion_matrix(y, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:03:13.881959Z","iopub.execute_input":"2025-03-31T11:03:13.882278Z","iopub.status.idle":"2025-03-31T11:03:13.980626Z","shell.execute_reply.started":"2025-03-31T11:03:13.882250Z","shell.execute_reply":"2025-03-31T11:03:13.979503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntest_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:19:10.439791Z","iopub.execute_input":"2025-03-31T12:19:10.440153Z","iopub.status.idle":"2025-03-31T12:19:11.929038Z","shell.execute_reply.started":"2025-03-31T12:19:10.440125Z","shell.execute_reply":"2025-03-31T12:19:11.928097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Importing the test set\n\nX_test = test_data.values\nX_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:03:15.414151Z","iopub.execute_input":"2025-03-31T11:03:15.414672Z","iopub.status.idle":"2025-03-31T11:03:15.420889Z","shell.execute_reply.started":"2025-03-31T11:03:15.414638Z","shell.execute_reply":"2025-03-31T11:03:15.419667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Scaling the test set\n\nX_test_scaled = scaler.fit_transform(X_test)\nX_test_scaled","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:03:19.134329Z","iopub.execute_input":"2025-03-31T11:03:19.134833Z","iopub.status.idle":"2025-03-31T11:03:19.557952Z","shell.execute_reply.started":"2025-03-31T11:03:19.134719Z","shell.execute_reply":"2025-03-31T11:03:19.556834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generating the predictions on the test set\n\npredictions = rf.predict(X_test_scaled)\npredictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:03:25.870572Z","iopub.execute_input":"2025-03-31T11:03:25.871105Z","iopub.status.idle":"2025-03-31T11:03:26.129860Z","shell.execute_reply.started":"2025-03-31T11:03:25.871045Z","shell.execute_reply":"2025-03-31T11:03:26.128901Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Saving the predictions in a dataframe\n\npred_df = pd.DataFrame(columns = ['ImageId', 'Label'])\npred_df['ImageId'] = np.arange(1, 28001)\npred_df['Label'] = predictions\npred_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:03:31.550602Z","iopub.execute_input":"2025-03-31T11:03:31.551007Z","iopub.status.idle":"2025-03-31T11:03:31.573761Z","shell.execute_reply.started":"2025-03-31T11:03:31.550976Z","shell.execute_reply":"2025-03-31T11:03:31.572636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Writing the predictions in a csv file\n\npred_df.to_csv('ml_submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:03:42.230496Z","iopub.execute_input":"2025-03-31T11:03:42.230862Z","iopub.status.idle":"2025-03-31T11:03:42.263738Z","shell.execute_reply.started":"2025-03-31T11:03:42.230834Z","shell.execute_reply":"2025-03-31T11:03:42.262567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Splitting the dataset into training set and validation set\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom sklearn.model_selection import train_test_split\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler,ReduceLROnPlateau\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.1)\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:16:26.331936Z","iopub.status.idle":"2025-03-31T12:16:26.332280Z","shell.execute_reply":"2025-03-31T12:16:26.332150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Reshaping the data for Image Generator (Rank = 4)\n\nX_train = X_train.reshape((37800, 28, 28, 1))\nX_valid = X_valid.reshape((4200, 28, 28, 1))\nX_train.shape, X_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:04:52.357502Z","iopub.execute_input":"2025-03-31T11:04:52.358254Z","iopub.status.idle":"2025-03-31T11:04:52.365828Z","shell.execute_reply.started":"2025-03-31T11:04:52.358213Z","shell.execute_reply":"2025-03-31T11:04:52.364452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Applying Data Augmentation\n\ndatagen = ImageDataGenerator(rotation_range=15,  zoom_range = 0.1,  width_shift_range=0.1,  height_shift_range=0.1)\ndatagen.fit(X_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:05:07.176909Z","iopub.execute_input":"2025-03-31T11:05:07.177255Z","iopub.status.idle":"2025-03-31T11:05:07.277776Z","shell.execute_reply.started":"2025-03-31T11:05:07.177224Z","shell.execute_reply":"2025-03-31T11:05:07.276804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Defining the model architecture\n\nmodel = keras.models.Sequential([\n    keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same', input_shape = [28, 28, 1]),\n    keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'),\n    keras.layers.MaxPool2D(),\n    keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same'),\n    keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'),\n    keras.layers.MaxPool2D(),\n    keras.layers.Flatten(),\n    keras.layers.Dropout(0.25),\n    keras.layers.Dense(128, activation = 'relu'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(10, activation = 'softmax')\n])\n\nmodel.compile(loss = 'sparse_categorical_crossentropy', optimizer = 'nadam', metrics = ['accuracy'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:05:18.193133Z","iopub.execute_input":"2025-03-31T11:05:18.193512Z","iopub.status.idle":"2025-03-31T11:05:18.345843Z","shell.execute_reply.started":"2025-03-31T11:05:18.193482Z","shell.execute_reply":"2025-03-31T11:05:18.344498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(datagen.flow(X_train, y_train, batch_size=128),\n                    epochs=20,\n                    validation_data=(X_valid, y_valid))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:07:09.583235Z","iopub.execute_input":"2025-03-31T11:07:09.583631Z","iopub.status.idle":"2025-03-31T11:34:18.125740Z","shell.execute_reply.started":"2025-03-31T11:07:09.583601Z","shell.execute_reply":"2025-03-31T11:34:18.124360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualizing the convergence\n\npd.DataFrame(history.history).plot(figsize = (8, 5))\nplt.gca().set_ylim(0, 1)\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:35:06.819246Z","iopub.execute_input":"2025-03-31T11:35:06.819649Z","iopub.status.idle":"2025-03-31T11:35:07.132255Z","shell.execute_reply.started":"2025-03-31T11:35:06.819612Z","shell.execute_reply":"2025-03-31T11:35:07.130689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nX_test_scaled = X_test_scaled.reshape((28000, 28, 28, 1))\ncnn_pred_aug = np.argmax(model.predict(X_test_scaled), axis=1)\ncnn_pred_aug\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:38:12.423381Z","iopub.execute_input":"2025-03-31T11:38:12.423884Z","iopub.status.idle":"2025-03-31T11:38:29.023295Z","shell.execute_reply.started":"2025-03-31T11:38:12.423845Z","shell.execute_reply":"2025-03-31T11:38:29.022049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Saving the predictions in a dataframe\n\npred_df_cnn = pd.DataFrame(columns = ['ImageId', 'Label'])\npred_df_cnn['ImageId'] = np.arange(1, 28001)\npred_df_cnn['Label'] = cnn_pred_aug\npred_df_cnn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T11:38:35.913218Z","iopub.execute_input":"2025-03-31T11:38:35.913623Z","iopub.status.idle":"2025-03-31T11:38:35.929779Z","shell.execute_reply.started":"2025-03-31T11:38:35.913591Z","shell.execute_reply":"2025-03-31T11:38:35.927895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Writing the predictions in a csv file\npred_df_cnn.to_csv('dl_submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:19:39.891561Z","iopub.execute_input":"2025-03-31T12:19:39.891953Z","iopub.status.idle":"2025-03-31T12:19:39.917463Z","shell.execute_reply.started":"2025-03-31T12:19:39.891920Z","shell.execute_reply":"2025-03-31T12:19:39.916526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# An empty list to store the ensemble of 7 CNNs\nmodel_list = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:07:14.898301Z","iopub.execute_input":"2025-03-31T12:07:14.898779Z","iopub.status.idle":"2025-03-31T12:07:14.903151Z","shell.execute_reply.started":"2025-03-31T12:07:14.898735Z","shell.execute_reply":"2025-03-31T12:07:14.902166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Learning rate annealer & earlyStopping\n\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\nreduce_lr = ReduceLROnPlateau(monitor='val_accuracy', \n                                patience=3, \n                                verbose=1, \n                                factor=0.2, \n                                min_lr=1e-6)\n\nearly_stop = EarlyStopping(monitor='val_accuracy', patience=5, restore_best_weights=True, verbose=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:07:53.483537Z","iopub.execute_input":"2025-03-31T12:07:53.484016Z","iopub.status.idle":"2025-03-31T12:07:53.489960Z","shell.execute_reply.started":"2025-03-31T12:07:53.483979Z","shell.execute_reply":"2025-03-31T12:07:53.488702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating 7 objects of the same CNN architecture and saving in model_list\nfor i in range(7):\n    model = keras.models.Sequential([\n    keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same', input_shape = [28, 28, 1]),\n    keras.layers.BatchNormalization(),\n    keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'),\n    keras.layers.MaxPool2D(),\n    keras.layers.Conv2D(32, kernel_size = 3, activation = 'relu', padding = 'same'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Conv2D(64, kernel_size = 3, activation = 'relu', padding = 'same'),\n    keras.layers.MaxPool2D(),\n    keras.layers.Flatten(),\n    keras.layers.Dropout(0.2),\n    keras.layers.Dense(256, activation = 'relu'),\n    keras.layers.Dropout(0.45),\n    keras.layers.Dense(10, activation = 'softmax')])\n\n    model.compile(loss = 'sparse_categorical_crossentropy', optimizer = 'nadam', metrics = ['accuracy'])\n    model_list.append(model)\n    \nmodel_list[0].summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:07:56.258507Z","iopub.execute_input":"2025-03-31T12:07:56.258851Z","iopub.status.idle":"2025-03-31T12:07:56.887385Z","shell.execute_reply.started":"2025-03-31T12:07:56.258824Z","shell.execute_reply":"2025-03-31T12:07:56.886049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:10:47.180296Z","iopub.execute_input":"2025-03-31T12:10:47.180718Z","iopub.status.idle":"2025-03-31T12:10:47.187432Z","shell.execute_reply.started":"2025-03-31T12:10:47.180684Z","shell.execute_reply":"2025-03-31T12:10:47.186389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = X.reshape((42000, 28, 28, 1))\nX.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:11:01.954912Z","iopub.execute_input":"2025-03-31T12:11:01.955252Z","iopub.status.idle":"2025-03-31T12:11:01.961290Z","shell.execute_reply.started":"2025-03-31T12:11:01.955225Z","shell.execute_reply":"2025-03-31T12:11:01.960366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = [0] * 7\n\nfor i in range(7):\n    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1)\n    history[i] = model_list[i].fit(\n        datagen.flow(X_train, y_train, batch_size=128), \n        epochs=30, \n        validation_data=(X_valid, y_valid), \n        callbacks=[reduce_lr, early_stop]  \n    )\n    print(\"CNN : {} Maximum Train Accuracy : {} Maximum Validation Accuracy : {}\".format(\n        i+1, max(history[i].history['accuracy']), max(history[i].history['val_accuracy'])\n    ))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T12:20:38.696314Z","iopub.execute_input":"2025-03-31T12:20:38.696834Z","iopub.status.idle":"2025-03-31T13:56:13.485294Z","shell.execute_reply.started":"2025-03-31T12:20:38.696793Z","shell.execute_reply":"2025-03-31T13:56:13.483958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating a prediction tensor for applying the bagging technique\n\nensemble_cnn_pred = np.zeros((X_test.shape[0], 10))\nensemble_cnn_pred.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T14:48:30.382750Z","iopub.execute_input":"2025-03-31T14:48:30.383964Z","iopub.status.idle":"2025-03-31T14:48:30.395972Z","shell.execute_reply.started":"2025-03-31T14:48:30.383871Z","shell.execute_reply":"2025-03-31T14:48:30.394819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generating and aggregating predictions\n\nfor i in range(7):\n  ensemble_cnn_pred = ensemble_cnn_pred + model_list[i].predict(X_test_scaled)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T14:48:36.650365Z","iopub.execute_input":"2025-03-31T14:48:36.650783Z","iopub.status.idle":"2025-03-31T14:50:37.757568Z","shell.execute_reply.started":"2025-03-31T14:48:36.650743Z","shell.execute_reply":"2025-03-31T14:50:37.756442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verifying the predictions of the ensemble\n\nnp.sum(ensemble_cnn_pred[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T14:50:45.698264Z","iopub.execute_input":"2025-03-31T14:50:45.698625Z","iopub.status.idle":"2025-03-31T14:50:45.706016Z","shell.execute_reply.started":"2025-03-31T14:50:45.698597Z","shell.execute_reply":"2025-03-31T14:50:45.704897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Aggregating the predictions\n\nensemble_cnn_pred = np.argmax(ensemble_cnn_pred, axis = 1)\nensemble_cnn_pred[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T14:50:48.345535Z","iopub.execute_input":"2025-03-31T14:50:48.345912Z","iopub.status.idle":"2025-03-31T14:50:48.353686Z","shell.execute_reply.started":"2025-03-31T14:50:48.345882Z","shell.execute_reply":"2025-03-31T14:50:48.352542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verifying the shape\n\nensemble_cnn_pred.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T14:50:51.169484Z","iopub.execute_input":"2025-03-31T14:50:51.169827Z","iopub.status.idle":"2025-03-31T14:50:51.175671Z","shell.execute_reply.started":"2025-03-31T14:50:51.169800Z","shell.execute_reply":"2025-03-31T14:50:51.174660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Saving the predictions in a dataframe\n\npred_df_ensemble_cnn = pd.DataFrame(columns = ['ImageId', 'Label'])\npred_df_ensemble_cnn['ImageId'] = np.arange(1, 28001)\npred_df_ensemble_cnn['Label'] = ensemble_cnn_pred\npred_df_ensemble_cnn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-31T14:50:54.338591Z","iopub.execute_input":"2025-03-31T14:50:54.338957Z","iopub.status.idle":"2025-03-31T14:50:54.363460Z","shell.execute_reply.started":"2025-03-31T14:50:54.338910Z","shell.execute_reply":"2025-03-31T14:50:54.362276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Writing the predictions in a csv file\n\npred_df_ensemble_cnn.to_csv('dl_ensemble_submission.csv', index = False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}