{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"},{"sourceId":45106,"sourceType":"datasetVersion","datasetId":34137}],"dockerImageVersionId":29995,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<font size=\"3\">\n    <div style=\"text-align: right\"> <b> Author </b> </div>\n</font>\n<div style=\"text-align: right\"> Jędrzej </div>\n<div style=\"text-align: right\"> Dudzicz </div>","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T13:13:10.111636Z","iopub.execute_input":"2025-12-03T13:13:10.111904Z","iopub.status.idle":"2025-12-03T13:13:10.126494Z","shell.execute_reply.started":"2025-12-03T13:13:10.111880Z","shell.execute_reply":"2025-12-03T13:13:10.125900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import required libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n\n# Suppress warnings\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nos.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'\nos.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport tensorflow as tf\nfrom tensorflow.python.client import device_lib\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam\n\nprint(\"=\"*60)\nprint(\"MNIST DIGIT RECOGNITION USING CNN\")\nprint(\"=\"*60)\nprint(\"\\nTensorFlow version:\", tf.__version__)\ngpu_available = any('GPU' in str(device) for device in device_lib.list_local_devices())\nprint(\"GPU Available:\", gpu_available)\n\n# ============================================\n# LOAD DATASET\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"LOADING DATASET\")\nprint(\"=\"*60)\n\n# Load training data\ntrain_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\nprint(\"\\nDataset Information:\")\nprint(f\"Training data shape: {train_data.shape}\")\nprint(f\"Test data shape: {test_data.shape}\")\nprint(f\"\\nTraining data columns: {train_data.columns.tolist()[:5]}...\")\n\n# ============================================\n# EXPLORATORY DATA ANALYSIS\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"EXPLORATORY DATA ANALYSIS\")\nprint(\"=\"*60)\n\n# Check for missing values\nprint(f\"\\nMissing values in training data: {train_data.isnull().sum().sum()}\")\nprint(f\"Missing values in test data: {test_data.isnull().sum().sum()}\")\n\n# Class distribution\nprint(\"\\nClass Distribution:\")\nclass_counts = train_data['label'].value_counts().sort_index()\nfor digit, count in class_counts.items():\n    print(f\"Digit {digit}: {count} samples ({count/len(train_data)*100:.1f}%)\")\n\n# Visualize class distribution\nplt.figure(figsize=(10, 5))\nsns.barplot(x=class_counts.index, y=class_counts.values, palette='viridis')\nplt.title('Distribution of Digits in Training Data', fontsize=14, fontweight='bold')\nplt.xlabel('Digit', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(range(10))\nplt.grid(axis='y', alpha=0.3)\nplt.tight_layout()\nplt.show()\n\n# Display sample images\nprint(\"\\nDisplaying sample images from each digit class...\")\nfig, axes = plt.subplots(2, 5, figsize=(15, 6))\nfig.suptitle('Sample Images for Each Digit (0-9)', fontsize=14, fontweight='bold')\naxes = axes.ravel()\n\nfor digit in range(10):\n    sample = train_data[train_data['label'] == digit].iloc[0, 1:].values.reshape(28, 28)\n    axes[digit].imshow(sample, cmap='gray')\n    axes[digit].set_title(f'Digit: {digit}', fontsize=11)\n    axes[digit].axis('off')\n\nplt.tight_layout()\nplt.show()\n\n# Pixel value statistics\nprint(\"\\nPixel Value Statistics:\")\npixel_data = train_data.iloc[:, 1:]\nprint(f\"Min pixel value: {pixel_data.min().min()}\")\nprint(f\"Max pixel value: {pixel_data.max().max()}\")\nprint(f\"Mean pixel value: {pixel_data.mean().mean():.2f}\")\nprint(f\"Std pixel value: {pixel_data.std().mean():.2f}\")\n\n# ============================================\n# DATA PREPROCESSING\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"DATA PREPROCESSING\")\nprint(\"=\"*60)\n\n# Separate features and labels\nX = train_data.iloc[:, 1:].values\ny = train_data.iloc[:, 0].values\nX_test_final = test_data.values\n\nprint(f\"\\nOriginal shapes:\")\nprint(f\"X (features): {X.shape}\")\nprint(f\"y (labels): {y.shape}\")\nprint(f\"X_test_final: {X_test_final.shape}\")\n\n# Reshape data to 28x28x1 for CNN input\nX = X.reshape(-1, 28, 28, 1)\nX_test_final = X_test_final.reshape(-1, 28, 28, 1)\n\n# Normalize pixel values to range 0-1\nX = X.astype('float32') / 255.0\nX_test_final = X_test_final.astype('float32') / 255.0\n\nprint(f\"\\nAfter normalization:\")\nprint(f\"Min pixel value: {X.min()}\")\nprint(f\"Max pixel value: {X.max()}\")\nprint(f\"Data type: {X.dtype}\")\n\n# Convert labels to categorical one-hot encoding\ny_categorical = keras.utils.to_categorical(y, 10)\n\nprint(f\"\\nPreprocessed shapes:\")\nprint(f\"X: {X.shape}\")\nprint(f\"y_categorical: {y_categorical.shape}\")\n\n# Split into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y_categorical, test_size=0.15, random_state=42, stratify=y\n)\n\nprint(f\"\\nData Split:\")\nprint(f\"Training samples: {X_train.shape[0]} ({X_train.shape[0]/len(X)*100:.1f}%)\")\nprint(f\"Validation samples: {X_val.shape[0]} ({X_val.shape[0]/len(X)*100:.1f}%)\")\n\n# ============================================\n# DATA AUGMENTATION\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"DATA AUGMENTATION SETUP\")\nprint(\"=\"*60)\n\n# Create data augmentation generator\ndatagen = ImageDataGenerator(\n    rotation_range=10,          # Rotate images by up to 10 degrees\n    width_shift_range=0.1,      # Shift images horizontally\n    height_shift_range=0.1,     # Shift images vertically\n    zoom_range=0.1,             # Zoom in/out\n    shear_range=0.1,            # Shear transformation\n    fill_mode='nearest'\n)\n\ndatagen.fit(X_train)\nprint(\"✓ Data augmentation configured successfully\")\nprint(\"  - Rotation: ±10 degrees\")\nprint(\"  - Shift: ±10%\")\nprint(\"  - Zoom: ±10%\")\nprint(\"  - Shear: ±10%\")\n\n# ============================================\n# BUILD CNN MODEL\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"BUILDING CNN ARCHITECTURE\")\nprint(\"=\"*60)\n\nkeras.backend.clear_session()\n\nmodel = keras.Sequential([\n    # First Convolutional Block\n    layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1), padding='same'),\n    layers.BatchNormalization(),\n    layers.Conv2D(32, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.MaxPooling2D((2, 2)),\n    layers.Dropout(0.25),\n    \n    # Second Convolutional Block\n    layers.Conv2D(64, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.Conv2D(64, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.MaxPooling2D((2, 2)),\n    layers.Dropout(0.25),\n    \n    # Third Convolutional Block\n    layers.Conv2D(128, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.Conv2D(128, (3, 3), activation='relu', padding='same'),\n    layers.BatchNormalization(),\n    layers.MaxPooling2D((2, 2)),\n    layers.Dropout(0.25),\n    \n    # Fully Connected Layers\n    layers.Flatten(),\n    layers.Dense(256, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.5),\n    layers.Dense(128, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.5),\n    \n    # Output Layer\n    layers.Dense(10, activation='softmax')\n], name='MNIST_CNN_Classifier')\n\nprint(\"\\nModel Architecture Summary:\")\nmodel.summary()\n\n# Compile model\noptimizer = Adam(lr=0.001)\nmodel.compile(\n    optimizer=optimizer,\n    loss='categorical_crossentropy',\n    metrics=['accuracy']\n)\n\nprint(\"\\n✓ Model compiled successfully\")\nprint(f\"  - Total parameters: {model.count_params():,}\")\nprint(f\"  - Optimizer: Adam (lr=0.001)\")\nprint(f\"  - Loss function: Categorical Crossentropy\")\n\n# ============================================\n# SETUP CALLBACKS\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"CONFIGURING TRAINING CALLBACKS\")\nprint(\"=\"*60)\n\n# Early stopping\nearly_stopping = EarlyStopping(\n    monitor='val_loss',\n    patience=10,\n    verbose=1\n)\n\n# Reduce learning rate on plateau\nreduce_lr = ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.5,\n    patience=5,\n    min_lr=1e-7,\n    verbose=1\n)\n\n# Model checkpoint\ncheckpoint = ModelCheckpoint(\n    'best_mnist_model.h5',\n    monitor='val_loss',\n    save_best_only=True,\n    verbose=1\n)\n\ncallbacks = [early_stopping, reduce_lr, checkpoint]\n\nprint(\"✓ Callbacks configured:\")\nprint(\"  - Early Stopping (patience=10)\")\nprint(\"  - Learning Rate Reduction (patience=5)\")\nprint(\"  - Model Checkpoint (save best)\")\n\n# ============================================\n# TRAIN THE MODEL\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING THE MODEL\")\nprint(\"=\"*60)\n\nhistory = model.fit_generator(\n    datagen.flow(X_train, y_train, batch_size=64),\n    steps_per_epoch=len(X_train) // 64,\n    epochs=5,\n    validation_data=(X_val, y_val),\n    callbacks=callbacks,\n    verbose=1\n)\n\nprint(\"\\n✓ Training completed successfully\")\n\n# Load best weights\nprint(\"\\nLoading best model weights...\")\nmodel.load_weights('best_mnist_model.h5')\nprint(\"✓ Best weights restored\")\n\n# ============================================\n# EVALUATE THE MODEL\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"MODEL EVALUATION\")\nprint(\"=\"*60)\n\n# Plot training history\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n\n# Accuracy plot\nepochs_range = range(1, len(history.history['accuracy']) + 1)\nax1.plot(epochs_range, history.history['accuracy'], 'b-o', label='Training Accuracy', linewidth=2)\nax1.plot(epochs_range, history.history['val_accuracy'], 'r-o', label='Validation Accuracy', linewidth=2)\nax1.set_title('Model Accuracy Over Epochs', fontsize=14, fontweight='bold')\nax1.set_xlabel('Epoch', fontsize=12)\nax1.set_ylabel('Accuracy', fontsize=12)\nax1.legend(fontsize=11)\nax1.grid(True, alpha=0.3)\n\n# Loss plot\nax2.plot(epochs_range, history.history['loss'], 'b-o', label='Training Loss', linewidth=2)\nax2.plot(epochs_range, history.history['val_loss'], 'r-o', label='Validation Loss', linewidth=2)\nax2.set_title('Model Loss Over Epochs', fontsize=14, fontweight='bold')\nax2.set_xlabel('Epoch', fontsize=12)\nax2.set_ylabel('Loss', fontsize=12)\nax2.legend(fontsize=11)\nax2.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()\n\n# Evaluate on validation set\nprint(\"\\nEvaluating model performance...\")\nval_loss, val_accuracy = model.evaluate(X_val, y_val, verbose=0)\n\nprint(f\"\\n{'='*60}\")\nprint(\"VALIDATION RESULTS\")\nprint(f\"{'='*60}\")\nprint(f\"Validation Accuracy: {val_accuracy*100:.2f}%\")\nprint(f\"Validation Loss: {val_loss:.4f}\")\n\n# Calculate overfitting metrics\nfinal_train_acc = history.history['accuracy'][-1]\nfinal_val_acc = history.history['val_accuracy'][-1]\noverfitting_gap = (final_train_acc - final_val_acc) * 100\n\nprint(f\"\\nFinal Training Accuracy: {final_train_acc*100:.2f}%\")\nprint(f\"Final Validation Accuracy: {final_val_acc*100:.2f}%\")\nprint(f\"Overfitting Gap: {overfitting_gap:.2f}%\")\n\nif overfitting_gap < 2:\n    print(\"✓ Excellent! Minimal overfitting detected.\")\nelif overfitting_gap < 4:\n    print(\"✓ Good! Low overfitting detected.\")\nelse:\n    print(\"⚠ Moderate overfitting detected.\")\n\n# Generate predictions on validation set\nprint(\"\\nGenerating predictions on validation set...\")\ny_pred = model.predict(X_val, verbose=0)\ny_pred_classes = np.argmax(y_pred, axis=1)\ny_val_classes = np.argmax(y_val, axis=1)\n\n# Classification Report\nprint(\"\\n\" + \"=\"*60)\nprint(\"CLASSIFICATION REPORT\")\nprint(\"=\"*60)\nprint(classification_report(y_val_classes, y_pred_classes, \n                          target_names=[f'Digit {i}' for i in range(10)]))\n\n# Confusion Matrix\nprint(\"\\nGenerating confusion matrix...\")\ncm = confusion_matrix(y_val_classes, y_pred_classes)\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n            xticklabels=range(10), yticklabels=range(10),\n            cbar_kws={'label': 'Count'})\nplt.title('Confusion Matrix - Validation Set', fontsize=16, fontweight='bold')\nplt.xlabel('Predicted Digit', fontsize=12)\nplt.ylabel('True Digit', fontsize=12)\nplt.tight_layout()\nplt.show()\n\n# Per-class accuracy\nprint(\"\\nPer-Class Accuracy:\")\nprint(\"-\" * 40)\nfor digit in range(10):\n    mask = y_val_classes == digit\n    if mask.sum() > 0:\n        class_acc = (y_pred_classes[mask] == y_val_classes[mask]).mean()\n        print(f\"Digit {digit}: {class_acc*100:.2f}%\")\n\n# Display sample predictions\nprint(\"\\nDisplaying sample predictions...\")\nfig, axes = plt.subplots(3, 5, figsize=(15, 9))\nfig.suptitle('Sample Predictions (Green=Correct, Red=Incorrect)', \n             fontsize=14, fontweight='bold')\naxes = axes.ravel()\n\n# Get random samples\nnp.random.seed(42)\nsample_indices = np.random.choice(len(X_val), 15, replace=False)\n\nfor idx, ax_idx in enumerate(sample_indices):\n    axes[idx].imshow(X_val[ax_idx].reshape(28, 28), cmap='gray')\n    true_digit = y_val_classes[ax_idx]\n    pred_digit = y_pred_classes[ax_idx]\n    confidence = y_pred[ax_idx][pred_digit] * 100\n    \n    color = 'green' if true_digit == pred_digit else 'red'\n    axes[idx].set_title(f'True: {true_digit}\\nPred: {pred_digit}\\nConf: {confidence:.1f}%',\n                       color=color, fontsize=10)\n    axes[idx].axis('off')\n\nplt.tight_layout()\nplt.show()\n\n# ============================================\n# GENERATE PREDICTIONS FOR TEST SET\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"GENERATING TEST SET PREDICTIONS\")\nprint(\"=\"*60)\n\nprint(\"\\nPredicting on test set...\")\ntest_predictions = model.predict(X_test_final, verbose=0)\ntest_pred_classes = np.argmax(test_predictions, axis=1)\n\n# Create submission file\nsubmission = pd.DataFrame({\n    'ImageId': range(1, len(test_pred_classes) + 1),\n    'Label': test_pred_classes\n})\n\nsubmission.to_csv('submission.csv', index=False)\nprint(f\"\\n✓ Submission file created: 'submission.csv'\")\nprint(f\"  - Total predictions: {len(test_pred_classes):,}\")\nprint(f\"  - File shape: {submission.shape}\")\n\n# Show prediction distribution\nprint(\"\\nTest Set Prediction Distribution:\")\npred_dist = pd.Series(test_pred_classes).value_counts().sort_index()\nfor digit, count in pred_dist.items():\n    print(f\"Digit {digit}: {count} predictions ({count/len(test_pred_classes)*100:.1f}%)\")\n\n# Visualize test predictions\nplt.figure(figsize=(10, 5))\nsns.barplot(x=pred_dist.index, y=pred_dist.values, palette='viridis')\nplt.title('Distribution of Predicted Digits in Test Set', fontsize=14, fontweight='bold')\nplt.xlabel('Predicted Digit', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xticks(range(10))\nplt.grid(axis='y', alpha=0.3)\nplt.tight_layout()\nplt.show()\n\n# Display sample test predictions\nprint(\"\\nDisplaying sample test set predictions...\")\nfig, axes = plt.subplots(3, 5, figsize=(15, 9))\nfig.suptitle('Sample Test Set Predictions', fontsize=14, fontweight='bold')\naxes = axes.ravel()\n\nfor i in range(15):\n    axes[i].imshow(X_test_final[i].reshape(28, 28), cmap='gray')\n    pred_digit = test_pred_classes[i]\n    confidence = test_predictions[i][pred_digit] * 100\n    axes[i].set_title(f'Predicted: {pred_digit}\\nConfidence: {confidence:.1f}%', fontsize=10)\n    axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()\n\n# ============================================\n# FINAL SUMMARY\n# ============================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"FINAL SUMMARY\")\nprint(\"=\"*60)\nprint(f\"\\n✓ Model Architecture: Deep CNN with Batch Normalization\")\nprint(f\"✓ Total Parameters: {model.count_params():,}\")\nprint(f\"✓ Training Samples: {len(X_train):,}\")\nprint(f\"✓ Validation Samples: {len(X_val):,}\")\nprint(f\"✓ Test Samples: {len(X_test_final):,}\")\nprint(f\"\\n✓ Validation Accuracy: {val_accuracy*100:.2f}%\")\nprint(f\"✓ Overfitting Gap: {overfitting_gap:.2f}%\")\nprint(f\"✓ Epochs Trained: {len(history.history['loss'])}\")\nprint(f\"\\n✓ Submission file ready: submission.csv\")\nprint(\"\\n\" + \"=\"*60)\nprint(\"ANALYSIS COMPLETE - MODEL READY FOR SUBMISSION!\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T13:30:37.615478Z","iopub.execute_input":"2025-12-03T13:30:37.615803Z","iopub.status.idle":"2025-12-03T13:31:38.605973Z","shell.execute_reply.started":"2025-12-03T13:30:37.615774Z","shell.execute_reply":"2025-12-03T13:31:38.605235Z"}},"outputs":[],"execution_count":null}]}