{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"fe51e124","cell_type":"markdown","source":"# MNIST Digit Recognition with Data Augmentation and Neural Networks\n\nThis notebook implements a digit recognition solution for the Kaggle Digit Recognizer competition using:\n- **Data Augmentation**: Image rotation, shifting, and zooming to increase training data diversity\n- **Neural Network**: Multi-layer Perceptron (MLP) with 3 hidden layers\n- **Advanced Training**: Early stopping and adaptive learning rate\n\n## Key Features:\n- Comprehensive data preprocessing\n- Custom augmentation functions\n- Progress tracking with visualization\n- Detailed performance metrics","metadata":{}},{"id":"3a9342d2","cell_type":"code","source":"# Import required libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport time\nfrom scipy.ndimage import rotate, shift, zoom\nfrom sklearn.utils import shuffle\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.metrics import accuracy_score\n\nprint(\"✅ All libraries imported successfully!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"db03d698","cell_type":"code","source":"# Load the training and test datasets\n# Note: Update paths as needed for Kaggle environment\ntrain_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\n# Display basic information about the training data\nprint(f\"Training data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")\nprint(f\"Training data columns: {list(train_df.columns)[:5]}...\")  # Show first 5 columns\n\n# Check for missing values\nprint(f\"\\nMissing values in training data: {train_df.isnull().sum().sum()}\")\nprint(f\"Missing values in test data: {test_df.isnull().sum().sum()}\")\n\n# Display first few rows\ntrain_df.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"68d2ef5b","cell_type":"code","source":"# Separate features (X) and target labels (y)\nX = train_df.drop(\"label\", axis=1)  # Remove the label column to get pixel values\ny = train_df[\"label\"]               # Extract the label column\n\nprint(f\"Feature matrix shape: {X.shape}\")\nprint(f\"Labels shape: {y.shape}\")\nprint(f\"Unique labels: {sorted(y.unique())}\")\nprint(f\"Label distribution:\")\nprint(y.value_counts().sort_index())","metadata":{},"outputs":[],"execution_count":null},{"id":"0853055a","cell_type":"code","source":"# Visualize some sample digits\nfig, axes = plt.subplots(2, 5, figsize=(15, 6))\nfor i in range(10):\n    row = i // 5\n    col = i % 5\n    image = X.iloc[i].values.reshape(28, 28)\n    axes[row, col].imshow(image, cmap='gray')\n    axes[row, col].set_title(f'Label: {y.iloc[i]}')\n    axes[row, col].axis('off')\n\nplt.suptitle('Sample Training Images', fontsize=16)\nplt.tight_layout()\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"id":"6d8eaf67","cell_type":"code","source":"# Normalize pixel values to range [0, 1] by dividing by 255\nX = X / 255.0\ntest_df = test_df / 255.0\n\nprint(f\"Pixel value range after normalization:\")\nprint(f\"Min: {X.min().min():.3f}, Max: {X.max().max():.3f}\")\nprint(f\"✅ Data normalization completed!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"f3834003","cell_type":"code","source":"# Data Augmentation Functions\ndef rotate_image(image, angle_range=15):\n    \"\"\"Rotate image by random angle within range\"\"\"\n    angle = np.random.uniform(-angle_range, angle_range)\n    image_2d = image.reshape(28, 28)\n    rotated = rotate(image_2d, angle, reshape=False, mode='constant', cval=0)\n    return rotated.flatten()\n\ndef shift_image(image, shift_range=2):\n    \"\"\"Shift image by random pixels within range\"\"\"\n    dx = np.random.uniform(-shift_range, shift_range)\n    dy = np.random.uniform(-shift_range, shift_range)\n    image_2d = image.reshape(28, 28)\n    shifted = shift(image_2d, [dy, dx], mode='constant', cval=0)\n    return shifted.flatten()\n\ndef zoom_image(image, zoom_range=0.1):\n    \"\"\"Zoom image by random factor within range\"\"\"\n    zoom_factor = np.random.uniform(1-zoom_range, 1+zoom_range)\n    image_2d = image.reshape(28, 28)\n    zoomed = zoom(image_2d, zoom_factor, mode='constant', cval=0)\n    \n    # Ensure the output is always 28x28\n    h, w = zoomed.shape\n    if h > 28 or w > 28:\n        # Crop to 28x28\n        start_h = (h - 28) // 2\n        start_w = (w - 28) // 2\n        zoomed = zoomed[start_h:start_h+28, start_w:start_w+28]\n    elif h < 28 or w < 28:\n        # Pad to 28x28\n        pad_h = (28 - h) // 2\n        pad_w = (28 - w) // 2\n        zoomed = np.pad(zoomed, ((pad_h, 28-h-pad_h), (pad_w, 28-w-pad_w)), \n                       mode='constant', constant_values=0)\n    \n    # Ensure exactly 28x28\n    zoomed = zoomed[:28, :28]\n    return zoomed.flatten()\n\nprint(\"✅ Data augmentation functions defined!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"fb51d54b","cell_type":"code","source":"def augment_data(X, y, augmentation_factor=2):\n    \"\"\"Generate augmented data\"\"\"\n    print(f\"Generating {augmentation_factor}x augmented data...\")\n    \n    X_augmented = []\n    y_augmented = []\n    \n    # Original data\n    X_augmented.append(X)\n    y_augmented.append(y)\n    \n    # Generate augmented samples\n    for i in tqdm(range(augmentation_factor), desc=\"Data Augmentation\"):\n        X_aug = []\n        for img in X.values:\n            # Randomly choose augmentation method\n            aug_method = np.random.choice(['rotate', 'shift', 'zoom'])\n            if aug_method == 'rotate':\n                aug_img = rotate_image(img)\n            elif aug_method == 'shift':\n                aug_img = shift_image(img)\n            else:  # zoom\n                aug_img = zoom_image(img)\n            \n            # Ensure the augmented image has the correct size\n            if len(aug_img) != 784:  # 28*28 = 784\n                print(f\"Warning: Augmented image has size {len(aug_img)}, expected 784\")\n                aug_img = aug_img[:784] if len(aug_img) > 784 else np.pad(aug_img, (0, 784-len(aug_img)))\n            \n            X_aug.append(aug_img)\n        \n        X_aug = np.array(X_aug)\n        X_augmented.append(X_aug)\n        y_augmented.append(y.values)\n    \n    # Combine all data\n    X_final = np.vstack(X_augmented)\n    y_final = np.hstack(y_augmented)\n    \n    # Shuffle the data\n    X_final, y_final = shuffle(X_final, y_final, random_state=42)\n    \n    print(f\"Original data size: {X.shape[0]}\")\n    print(f\"Augmented data size: {X_final.shape[0]}\")\n    \n    return X_final, y_final\n\n# Apply data augmentation to training data\nX_augmented, y_augmented = augment_data(X, y, augmentation_factor=2)","metadata":{},"outputs":[],"execution_count":null},{"id":"2baa8814","cell_type":"code","source":"def show_augmentation_examples(X_original, X_augmented, y_augmented, num_examples=5):\n    \"\"\"Show examples of original and augmented images\"\"\"\n    fig, axes = plt.subplots(2, num_examples, figsize=(15, 6))\n    \n    for i in range(num_examples):\n        # Original image\n        original_img = X_original.iloc[i].values.reshape(28, 28)\n        axes[0, i].imshow(original_img, cmap='gray')\n        axes[0, i].set_title(f'Original\\nLabel: {y.iloc[i]}')\n        axes[0, i].axis('off')\n        \n        # Find an augmented version of the same digit\n        same_label_indices = np.where(y_augmented == y.iloc[i])[0]\n        # Skip the original (first occurrence)\n        if len(same_label_indices) > 1:\n            aug_idx = same_label_indices[1]  # Take the first augmented version\n            aug_img = X_augmented[aug_idx].reshape(28, 28)\n            axes[1, i].imshow(aug_img, cmap='gray')\n            axes[1, i].set_title(f'Augmented\\nLabel: {y_augmented[aug_idx]}')\n        else:\n            axes[1, i].text(0.5, 0.5, 'No augmented\\nversion found', \n                          horizontalalignment='center', verticalalignment='center')\n        axes[1, i].axis('off')\n    \n    plt.suptitle('Data Augmentation Examples', fontsize=16)\n    plt.tight_layout()\n    plt.show()\n\n# Show augmentation examples\nshow_augmentation_examples(X, X_augmented, y_augmented)","metadata":{},"outputs":[],"execution_count":null},{"id":"cac2a12f","cell_type":"code","source":"# Further standardize the features for neural network\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_augmented)\ntest_df_scaled = scaler.transform(test_df)\n\nprint(f\"Feature scaling completed!\")\nprint(f\"Scaled training data shape: {X_scaled.shape}\")\nprint(f\"Scaled test data shape: {test_df_scaled.shape}\")\nprint(f\"Scaled feature statistics:\")\nprint(f\"Mean: {X_scaled.mean():.6f}, Std: {X_scaled.std():.6f}\")","metadata":{},"outputs":[],"execution_count":null},{"id":"437124b7","cell_type":"code","source":"# Split the training data into training and validation sets (80/20 split)\nX_train, X_val, y_train, y_val = train_test_split(\n    X_scaled, y_augmented,\n    test_size=0.2,\n    random_state=42,\n    stratify=y_augmented  # Ensure balanced split across all digit classes\n)\n\nprint(f\"Training set size: {X_train.shape[0]} samples\")\nprint(f\"Validation set size: {X_val.shape[0]} samples\")\nprint(f\"Training set shape: {X_train.shape}\")\nprint(f\"Validation set shape: {X_val.shape}\")\n\n# Check class distribution in train/val sets\nprint(f\"\\nClass distribution in training set:\")\nprint(pd.Series(y_train).value_counts().sort_index())\nprint(f\"\\nClass distribution in validation set:\")\nprint(pd.Series(y_val).value_counts().sort_index())","metadata":{},"outputs":[],"execution_count":null},{"id":"1654b28d","cell_type":"code","source":"# Initialize and configure the Neural Network (Multi-layer Perceptron) model\nmodel = MLPClassifier(\n    hidden_layer_sizes=(128, 64, 32),  # 3 hidden layers: 128, 64, 32 neurons\n    activation='relu',                  # ReLU activation function\n    solver='adam',                      # Adam optimizer\n    alpha=0.001,                        # L2 regularization parameter\n    batch_size='auto',                  # Automatic batch size adjustment\n    learning_rate='adaptive',           # Adaptive learning rate\n    max_iter=500,                       # Maximum number of iterations\n    random_state=42,                    # Random seed for reproducibility\n    early_stopping=True,                # Early stopping\n    validation_fraction=0.1,            # Validation fraction for early stopping\n    n_iter_no_change=10,                # Number of iterations with no improvement threshold\n    verbose=True                        # Detailed output during training\n)\n\n# Train the model on the training data\nprint(\"🚀 Starting neural network training...\")\nprint(f\"Maximum epochs: {model.max_iter}\")\nprint(f\"Training data size: {X_train.shape}\")\nprint(f\"Validation data size: {X_val.shape}\")\nprint(\"-\" * 50)\n\n# Start training time measurement\nstart_time = time.time()\n\n# Training with progress display using tqdm\nwith tqdm(total=model.max_iter, desc=\"Training Progress\", \n          bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]') as pbar:\n    \n    # Model training\n    model.fit(X_train, y_train)\n    pbar.n = model.n_iter_\n    pbar.refresh()\n\n# Calculate training time\ntraining_time = time.time() - start_time\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"🎉 Training completed!\")\nprint(f\"Actual epochs: {model.n_iter_}\")\nprint(f\"Training time: {training_time:.2f} seconds\")\nprint(f\"Final loss: {model.loss_:.6f}\")\nprint(\"=\"*50)","metadata":{},"outputs":[],"execution_count":null},{"id":"aaaf801e","cell_type":"code","source":"# Make predictions on the validation set and calculate accuracy\nprint(\"📊 Evaluating model on validation data...\")\nwith tqdm(total=1, desc=\"Validation\", bar_format='{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt}') as pbar:\n    y_pred = model.predict(X_val)\n    pbar.update(1)\n\nacc = accuracy_score(y_val, y_pred)\n\nprint(f\"\\n🎯 Validation Results:\")\nprint(f\"   Validation Accuracy: {acc:.4f} ({acc*100:.2f}%)\")\nprint(f\"   Correct Predictions: {np.sum(y_pred == y_val)} / {len(y_val)}\")\n\n# Detailed classification metrics\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nprint(f\"\\n📈 Detailed Classification Report:\")\nprint(classification_report(y_val, y_pred))\n\n# Confusion matrix visualization\nimport seaborn as sns\n\nplt.figure(figsize=(10, 8))\ncm = confusion_matrix(y_val, y_pred)\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n            xticklabels=range(10), yticklabels=range(10))\nplt.title('Confusion Matrix - Validation Set')\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.show()\n\n# Calculate per-class accuracy\nper_class_acc = cm.diagonal() / cm.sum(axis=1)\nprint(f\"\\n🔍 Per-class Accuracy:\")\nfor i, acc_val in enumerate(per_class_acc):\n    print(f\"   Digit {i}: {acc_val:.3f} ({acc_val*100:.1f}%)\")","metadata":{},"outputs":[],"execution_count":null},{"id":"386ca0f8","cell_type":"code","source":"# Generate predictions on the test set\nprint(\"🔮 Generating predictions on test data...\")\nwith tqdm(total=1, desc=\"Test Prediction\", bar_format='{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt}') as pbar:\n    test_preds = model.predict(test_df_scaled)\n    pbar.update(1)\n\nprint(f\"Test predictions generated: {len(test_preds)} samples\")\nprint(f\"Prediction distribution:\")\nprint(pd.Series(test_preds).value_counts().sort_index())\n\n# Create submission file in the required format\nsubmission = pd.DataFrame({\n    \"ImageId\": np.arange(1, len(test_preds) + 1),  # Image IDs starting from 1\n    \"Label\": test_preds                            # Predicted digit labels\n})\n\nprint(f\"\\n📋 Submission file preview:\")\nprint(submission.head(10))\nprint(f\"\\nSubmission file shape: {submission.shape}\")\n\n# Save the submission file as CSV\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(f\"\\n✅ Submission file saved as 'submission.csv'\")\nprint(f\"📁 File size: {len(submission)} rows\")\nprint(f\"🏆 Ready for Kaggle submission!\")","metadata":{},"outputs":[],"execution_count":null},{"id":"a4cabccb","cell_type":"markdown","source":"## 📊 Summary and Results\n\n### Model Architecture:\n- **Type**: Multi-layer Perceptron (MLP)\n- **Hidden Layers**: 3 layers (128 → 64 → 32 neurons)\n- **Activation**: ReLU\n- **Optimizer**: Adam with adaptive learning rate\n- **Regularization**: L2 (α=0.001) + Early stopping\n\n### Data Augmentation:\n- **Techniques**: Random rotation (±15°), shifting (±2 pixels), zooming (±10%)\n- **Augmentation Factor**: 2x (tripled dataset size)\n- **Benefits**: Improved generalization and reduced overfitting\n\n### Performance Metrics:\n- **Validation Accuracy**: Displayed above\n- **Training Time**: Efficient with early stopping\n- **Robust Predictions**: Enhanced by data augmentation\n\n### Key Features:\n1. **Comprehensive preprocessing** with normalization and standardization\n2. **Advanced augmentation** to increase data diversity\n3. **Progress tracking** with visual feedback\n4. **Detailed evaluation** with confusion matrix and per-class metrics\n5. **Production-ready submission** format\n\n### Next Steps:\n- Consider deeper architectures (CNN)\n- Experiment with different augmentation parameters\n- Ensemble methods for improved accuracy\n- Hyperparameter tuning for optimization","metadata":{}}]}