{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":99236,"databundleVersionId":11855598,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:08.881691Z","iopub.execute_input":"2025-06-01T04:23:08.882586Z","iopub.status.idle":"2025-06-01T04:23:09.273162Z","shell.execute_reply.started":"2025-06-01T04:23:08.882546Z","shell.execute_reply":"2025-06-01T04:23:09.271901Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 1. Import Required Libraries\n# ==============================","metadata":{}},{"cell_type":"code","source":"import random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:09.274613Z","iopub.execute_input":"2025-06-01T04:23:09.274987Z","iopub.status.idle":"2025-06-01T04:23:27.940083Z","shell.execute_reply.started":"2025-06-01T04:23:09.274961Z","shell.execute_reply":"2025-06-01T04:23:27.939275Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 2. Load CSV Data\n# ==============================","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/exploring-machine-learning-with-mnist/mnist_train.csv')\ntest_df = pd.read_csv('/kaggle/input/exploring-machine-learning-with-mnist/mnist_test.csv')\n\nprint(\"Train shape:\", train_df.shape)  # (60000, 785)\nprint(\"Test shape:\", test_df.shape)    # (10000, 785)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:27.941025Z","iopub.execute_input":"2025-06-01T04:23:27.941621Z","iopub.status.idle":"2025-06-01T04:23:34.046550Z","shell.execute_reply.started":"2025-06-01T04:23:27.941593Z","shell.execute_reply":"2025-06-01T04:23:34.045603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 3. Split Features and Labels\n# ==============================","metadata":{}},{"cell_type":"code","source":"X_train = train_df.drop('label', axis=1).values\ny_train = train_df['label'].values\n\nX_test = test_df.drop('label', axis=1).values\ny_test = test_df['label'].values \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:34.048567Z","iopub.execute_input":"2025-06-01T04:23:34.048860Z","iopub.status.idle":"2025-06-01T04:23:34.237720Z","shell.execute_reply.started":"2025-06-01T04:23:34.048837Z","shell.execute_reply":"2025-06-01T04:23:34.236846Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 4. Normalize and Reshape Data\n# ==============================","metadata":{}},{"cell_type":"code","source":"X_train = X_train.astype('float32') / 255.0\nX_test = X_test.astype('float32') / 255.0\n\nX_train = X_train.reshape(-1, 28, 28, 1)\nX_test = X_test.reshape(-1, 28, 28, 1)\n\n# One-hot encode labels for training and test data\ny_train_cat = tf.keras.utils.to_categorical(y_train, 10)\ny_test_cat = tf.keras.utils.to_categorical(y_test, 10)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:34.238570Z","iopub.execute_input":"2025-06-01T04:23:34.238870Z","iopub.status.idle":"2025-06-01T04:23:34.432474Z","shell.execute_reply.started":"2025-06-01T04:23:34.238848Z","shell.execute_reply":"2025-06-01T04:23:34.431603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 5. Visualize Sample Images\n# ==============================","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6, 4))\nrandom_indices = np.random.choice(len(X_train), 6, replace=False)\n\nfor i, idx in enumerate(random_indices):\n    plt.subplot(2, 3, i + 1)\n    plt.imshow(X_train[idx].reshape(28, 28), cmap='gray')\n    plt.title(f\"Label: {y_train[idx]}\")\n    plt.axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:34.433502Z","iopub.execute_input":"2025-06-01T04:23:34.433828Z","iopub.status.idle":"2025-06-01T04:23:34.870357Z","shell.execute_reply.started":"2025-06-01T04:23:34.433801Z","shell.execute_reply":"2025-06-01T04:23:34.869213Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 6. Create Validation Set\n# ==============================","metadata":{}},{"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n\nfor train_idx, val_idx in sss.split(X_train, y_train):\n    X_train, X_val = X_train[train_idx], X_train[val_idx]\n    y_train_cat, y_val_cat = y_train_cat[train_idx], y_train_cat[val_idx]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:34.871359Z","iopub.execute_input":"2025-06-01T04:23:34.871904Z","iopub.status.idle":"2025-06-01T04:23:35.415880Z","shell.execute_reply.started":"2025-06-01T04:23:34.871878Z","shell.execute_reply":"2025-06-01T04:23:35.414747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**StratifiedShuffleSplit:**  \n - **Purpose:** Performs a randomized split of the dataset while preserving the class distribution in both training and validation sets.\n\n**Stratified Sampling:**  \n - Ensures each class appears approximately in the same proportion in both X_train and X_val.\n - This is particularly important in datasets with class imbalance, helping the model generalize better and avoid bias during training.","metadata":{}},{"cell_type":"markdown","source":"# ==============================\n# 7. Data Augmentation with ImageDataGenerator\n# ==============================","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n    rotation_range=10,         # rotate images ±10 degrees\n    width_shift_range=0.1,     # shift images horizontally by 10%\n    height_shift_range=0.1,    # shift images vertically by 10%\n    zoom_range=0.1             # zoom in/out by 10%\n)\n\n# Fit the Generator on Training Data\ndatagen.fit(X_train)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:35.416825Z","iopub.execute_input":"2025-06-01T04:23:35.417122Z","iopub.status.idle":"2025-06-01T04:23:35.477704Z","shell.execute_reply.started":"2025-06-01T04:23:35.417100Z","shell.execute_reply":"2025-06-01T04:23:35.476750Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 8. Define Custom CNN Architecture\n# ==============================","metadata":{}},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    # Block 1\n    layers.Conv2D(32, (3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)),\n    layers.BatchNormalization(),\n    layers.Conv2D(32, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.MaxPooling2D((2, 2)),\n    layers.Dropout(0.25),\n\n    # Block 2\n    layers.Conv2D(64, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.Conv2D(64, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.MaxPooling2D((2, 2)),\n    layers.Dropout(0.25),\n\n    # Dense Layers\n    layers.Flatten(),\n    layers.Dense(256, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.5),\n    layers.Dense(10, activation='softmax')\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:35.478683Z","iopub.execute_input":"2025-06-01T04:23:35.478962Z","iopub.status.idle":"2025-06-01T04:23:35.663902Z","shell.execute_reply.started":"2025-06-01T04:23:35.478940Z","shell.execute_reply":"2025-06-01T04:23:35.662957Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 9. Compile the Model\n# ==============================","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:35.666675Z","iopub.execute_input":"2025-06-01T04:23:35.666966Z","iopub.status.idle":"2025-06-01T04:23:35.706004Z","shell.execute_reply.started":"2025-06-01T04:23:35.666942Z","shell.execute_reply":"2025-06-01T04:23:35.705205Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 10. Define Callbacks\n# ==============================","metadata":{}},{"cell_type":"code","source":"callbacks = [\n    ReduceLROnPlateau(monitor='val_accuracy', patience=3, factor=0.5, verbose=1),\n    EarlyStopping(monitor='val_loss', patience=6, restore_best_weights=True, verbose=1),\n    ModelCheckpoint('best_model.h5', monitor='val_loss', save_best_only=True, verbose=1)\n]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:35.706791Z","iopub.execute_input":"2025-06-01T04:23:35.707033Z","iopub.status.idle":"2025-06-01T04:23:35.712090Z","shell.execute_reply.started":"2025-06-01T04:23:35.707014Z","shell.execute_reply":"2025-06-01T04:23:35.711045Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**ReduceLROnPlateau:** Useful for stabilizing training when the model stops learning effectively at the current rate.  \n - **Purpose:** Dynamically reduces the learning rate when validation accuracy stagnates.\n - **Effect:** Helps the model escape plateaus by allowing finer convergence after initial rapid learning.\n\n**EarlyStopping:** Strong defense against overfitting. The use of restore_best_weights ensures the final model is optimal.  \n - **Purpose:** Stops training early if validation loss doesn’t improve for a set number of epochs.\n - **Effect:** Prevents overfitting and saves compute time.\n\n**ModelCheckpoint:** Helps with reproducibility and avoids relying only on in-memory weights.  \n - **Purpose:** Saves the model with the lowest validation loss during training.\n - **Effect:** Ensures that you always retain the best-performing model.","metadata":{}},{"cell_type":"markdown","source":"# ==============================\n# 11. Train the Model\n# ==============================","metadata":{}},{"cell_type":"code","source":"history = model.fit(datagen.flow(X_train, y_train_cat, batch_size=64),\n                    validation_data=(X_val, y_val_cat),\n                    epochs=20,\n                    callbacks=callbacks,\n                    verbose=2)\n\n# Save training history to JSON\nwith open(\"training_history.json\", \"w\") as f:\n    json.dump(history.history, f)\n\nprint(\"Training history saved to 'training_history.json'\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T04:23:35.713148Z","iopub.execute_input":"2025-06-01T04:23:35.713453Z","iopub.status.idle":"2025-06-01T05:03:33.545843Z","shell.execute_reply.started":"2025-06-01T04:23:35.713432Z","shell.execute_reply":"2025-06-01T05:03:33.545034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 11. Plot Training History\n# ==============================","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\n\n# Accuracy\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Train')\nplt.plot(history.history['val_accuracy'], label='Val')\nplt.title('Model Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\n\n# Loss\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], label='Train')\nplt.plot(history.history['val_loss'], label='Val')\nplt.title('Model Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:33.547585Z","iopub.execute_input":"2025-06-01T05:03:33.547945Z","iopub.status.idle":"2025-06-01T05:03:33.974858Z","shell.execute_reply.started":"2025-06-01T05:03:33.547915Z","shell.execute_reply":"2025-06-01T05:03:33.974016Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 12. Loading the Best Model\n# ==============================","metadata":{}},{"cell_type":"code","source":"# Load the best model saved during training\nmodel = load_model(\"/kaggle/working/best_model.h5\")\nprint(\"Best model loaded successfully.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:33.975899Z","iopub.execute_input":"2025-06-01T05:03:33.976223Z","iopub.status.idle":"2025-06-01T05:03:34.202201Z","shell.execute_reply.started":"2025-06-01T05:03:33.976195Z","shell.execute_reply":"2025-06-01T05:03:34.201184Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 12. Evaluate Model on Test Set\n# ==============================","metadata":{}},{"cell_type":"code","source":"test_loss, test_accuracy = model.evaluate(X_test, y_test_cat, verbose=0)\nprint(f\"Test Accuracy: {test_accuracy:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:34.203174Z","iopub.execute_input":"2025-06-01T05:03:34.203735Z","iopub.status.idle":"2025-06-01T05:03:40.768746Z","shell.execute_reply.started":"2025-06-01T05:03:34.203704Z","shell.execute_reply":"2025-06-01T05:03:40.767371Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 12. Predict on Test Set\n# ==============================","metadata":{}},{"cell_type":"code","source":"pred_probs = model.predict(X_test)\npred_labels = np.argmax(pred_probs, axis=1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:40.769938Z","iopub.execute_input":"2025-06-01T05:03:40.770249Z","iopub.status.idle":"2025-06-01T05:03:47.515261Z","shell.execute_reply.started":"2025-06-01T05:03:40.770225Z","shell.execute_reply":"2025-06-01T05:03:47.514192Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 13. Compute AUC-ROC Per Class\n# ==============================","metadata":{}},{"cell_type":"code","source":"# Confirm shape\nprint(\"pred_probs shape:\", pred_probs.shape)\nprint(\"y_test shape:\", y_test_cat.shape)\n\n# Compute per-class AUC\nauc_scores = []\nfor i in range(10):\n    auc = roc_auc_score(y_test_cat[:, i], pred_probs[:, i])\n    auc_scores.append(auc)\n    print(f\"Class {i} AUC: {auc:.4f}\")\n    \n# Compute overall macro-average\nmacro_auc = np.mean(auc_scores)\nprint(f\"Macro-average AUC: {macro_auc:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:47.516323Z","iopub.execute_input":"2025-06-01T05:03:47.516607Z","iopub.status.idle":"2025-06-01T05:03:47.577142Z","shell.execute_reply.started":"2025-06-01T05:03:47.516584Z","shell.execute_reply":"2025-06-01T05:03:47.576271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 14. Classification Report & Confusion Matrix\n# ==============================","metadata":{}},{"cell_type":"code","source":"# Classification report\nprint(\"Classification Report:\\n\")\nprint(classification_report(y_test, pred_labels))\n\n# Confusion matrix\ncm = confusion_matrix(y_test, pred_labels)\n# Plot confusion matrix\nplt.figure(figsize=(8,6))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n            xticklabels=range(10), yticklabels=range(10))\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.title('Confusion Matrix')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:47.578139Z","iopub.execute_input":"2025-06-01T05:03:47.578441Z","iopub.status.idle":"2025-06-01T05:03:48.024008Z","shell.execute_reply.started":"2025-06-01T05:03:47.578420Z","shell.execute_reply":"2025-06-01T05:03:48.023044Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 15. Visualize Predictions on Test Images\n# ==============================","metadata":{}},{"cell_type":"code","source":"def visualize_random_samples(X, y_true, y_pred=None):\n    \"\"\"\n    Displays 6 randomly selected grayscale images along \n    with their true and optional predicted labels.\n    \n    Args:\n        X (ndarray): Array of image data.\n        y_true (ndarray): True labels.\n        y_pred (ndarray, optional): Predicted probabilities or labels. Defaults to None.\n    \"\"\"\n    plt.figure(figsize=(12, 8))\n    indices = np.random.choice(len(X), 6, replace=False)\n    for i, idx in enumerate(indices):\n        plt.subplot(2, 3, i + 1)\n        plt.imshow(X[idx].reshape(28, 28), cmap='gray')\n        actual_label = y_true[idx]\n        pred_label = np.argmax(y_pred[idx]) if y_pred is not None else None\n        title = f\"Actual: {actual_label}\"\n        if y_pred is not None:\n            title += f\"\\nPred: {pred_label}\"\n        plt.title(title)\n        plt.axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Visualize random 6 images with actual and predicted labels\nvisualize_random_samples(X_test, y_test, pred_probs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:48.025129Z","iopub.execute_input":"2025-06-01T05:03:48.025498Z","iopub.status.idle":"2025-06-01T05:03:48.559569Z","shell.execute_reply.started":"2025-06-01T05:03:48.025469Z","shell.execute_reply":"2025-06-01T05:03:48.558569Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ==============================\n# 16. Prepare Submission File\n# ==============================","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'ID': test_df.index + 1,\n    'Label': list(pred_labels)\n})\n\n# Save to CSV\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file 'submission.csv' created.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T05:03:48.560451Z","iopub.execute_input":"2025-06-01T05:03:48.560733Z","iopub.status.idle":"2025-06-01T05:03:48.603426Z","shell.execute_reply.started":"2025-06-01T05:03:48.560712Z","shell.execute_reply":"2025-06-01T05:03:48.602326Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# =====================================\n# Overall Observations:\n - The model performs well with ~99% test accuracy.\n - Augmentation, regularization, and callbacks helped control overfitting.\n - Confusion matrix and classification report show good class-wise performance.\n - AUC is useful to understand probabilistic confidence for each class.\n# =====================================","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}