{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-22T06:03:46.158618Z","iopub.execute_input":"2025-07-22T06:03:46.158768Z","iopub.status.idle":"2025-07-22T06:03:47.832408Z","shell.execute_reply.started":"2025-07-22T06:03:46.158752Z","shell.execute_reply":"2025-07-22T06:03:47.831688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nimport seaborn as sns\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D, Input\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications import ResNet50V2, EfficientNetB0, MobileNetV2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T06:03:47.834295Z","iopub.execute_input":"2025-07-22T06:03:47.834651Z","iopub.status.idle":"2025-07-22T06:04:04.198308Z","shell.execute_reply.started":"2025-07-22T06:03:47.834630Z","shell.execute_reply":"2025-07-22T06:04:04.197697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.random.seed(42)\ntf.random.set_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T06:04:04.198896Z","iopub.execute_input":"2025-07-22T06:04:04.199355Z","iopub.status.idle":"2025-07-22T06:04:04.203515Z","shell.execute_reply.started":"2025-07-22T06:04:04.199337Z","shell.execute_reply":"2025-07-22T06:04:04.202918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PretrainedDigitRecognizer:\n    def __init__(self):\n        self.model = None\n        self.scaler = StandardScaler()\n        self.history = None\n        \n    def load_data(self, train_path='train.csv', test_path='test.csv'):\n        \"\"\"Load and preprocess the data\"\"\"\n        print(\"Loading data...\")\n        \n        # Load training data\n        train_data = pd.read_csv(train_path)\n        self.X_train = train_data.drop('label', axis=1).values\n        self.y_train = train_data['label'].values\n        \n        # Load test data\n        self.X_test = pd.read_csv(test_path).values\n        \n        print(f\"Training data shape: {self.X_train.shape}\")\n        print(f\"Test data shape: {self.X_test.shape}\")\n        \n        return self\n        \n    def preprocess_data(self, input_size=224):\n        \"\"\"Preprocess the data for pretrained models\"\"\"\n        print(\"Preprocessing data for pretrained models...\")\n        \n        # Normalize pixel values to [0, 1]\n        self.X_train = self.X_train / 255.0\n        self.X_test = self.X_test / 255.0\n        \n        # Reshape from 28x28 to target size (usually 224x224 for pretrained models)\n        self.input_size = input_size\n        \n        # First reshape to 28x28x1\n        X_train_temp = self.X_train.reshape(-1, 28, 28, 1)\n        X_test_temp = self.X_test.reshape(-1, 28, 28, 1)\n        \n        # Resize to input_size x input_size and convert to RGB (3 channels)\n        self.X_train = np.zeros((X_train_temp.shape[0], input_size, input_size, 3))\n        self.X_test = np.zeros((X_test_temp.shape[0], input_size, input_size, 3))\n        \n        for i in range(X_train_temp.shape[0]):\n            # Resize using tensorflow\n            resized = tf.image.resize(X_train_temp[i], (input_size, input_size))\n            # Convert grayscale to RGB by repeating the channel 3 times\n            self.X_train[i] = tf.repeat(resized, 3, axis=-1)\n            \n        for i in range(X_test_temp.shape[0]):\n            resized = tf.image.resize(X_test_temp[i], (input_size, input_size))\n            self.X_test[i] = tf.repeat(resized, 3, axis=-1)\n        \n        # Convert labels to categorical\n        self.y_train = to_categorical(self.y_train, 10)\n        \n        # Split training data for validation\n        self.X_train_split, self.X_val, self.y_train_split, self.y_val = train_test_split(\n            self.X_train, self.y_train, test_size=0.1, random_state=42, stratify=self.y_train\n        )\n        \n        print(f\"Training set: {self.X_train_split.shape}\")\n        print(f\"Validation set: {self.X_val.shape}\")\n        \n        return self\n        \n    def create_pretrained_model(self, model_name='resnet50v2', input_size=224):\n        \"\"\"Create pretrained model\"\"\"\n        \n        # Define base models\n        base_models = {\n            'resnet50v2': ResNet50V2,\n            'efficientnet': EfficientNetB0,\n            'mobilenet': MobileNetV2\n        }\n        \n        if model_name not in base_models:\n            raise ValueError(f\"Model {model_name} not supported. Choose from {list(base_models.keys())}\")\n        \n        # Create base model\n        base_model = base_models[model_name](\n            weights='imagenet',\n            include_top=False,\n            input_shape=(input_size, input_size, 3)\n        )\n        \n        # Freeze base model layers initially\n        base_model.trainable = False\n        \n        # Add custom classification head\n        inputs = Input(shape=(input_size, input_size, 3))\n        x = base_model(inputs, training=False)\n        x = GlobalAveragePooling2D()(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.5)(x)\n        x = Dense(512, activation='relu')(x)\n        x = BatchNormalization()(x)\n        x = Dropout(0.3)(x)\n        x = Dense(256, activation='relu')(x)\n        x = Dropout(0.2)(x)\n        outputs = Dense(10, activation='softmax')(x)\n        \n        model = Model(inputs, outputs)\n        \n        return model, base_model\n        \n    def fine_tune_model(self, base_model, fine_tune_at=100):\n        \"\"\"Fine-tune the pretrained model\"\"\"\n        # Unfreeze the top layers of the model\n        base_model.trainable = True\n        \n        # Fine-tune from this layer onwards\n        for layer in base_model.layers[:fine_tune_at]:\n            layer.trainable = False\n            \n        # Use a lower learning rate for fine-tuning\n        self.model.compile(\n            optimizer=Adam(learning_rate=0.00001),  \n            loss='categorical_crossentropy',\n            metrics=['accuracy']  \n        )\n        \n    def setup_data_augmentation(self):\n        \"\"\"Setup data augmentation\"\"\"\n        datagen = ImageDataGenerator(\n            rotation_range=10,\n            zoom_range=0.1,\n            width_shift_range=0.1,\n            height_shift_range=0.1,\n            shear_range=0.1,\n            horizontal_flip=False,  # Don't flip digits\n            fill_mode='nearest'\n        )\n        \n        return datagen\n    \n    def safe_combine_histories(self, history1, history2):\n        \"\"\"Safely combine training histories with error handling\"\"\"\n        combined_history = {}\n        \n        # Get available keys from both histories\n        keys1 = set(history1.history.keys())\n        keys2 = set(history2.history.keys()) if history2 else set()\n        \n        print(f\"Phase 1 history keys: {keys1}\")\n        if history2:\n            print(f\"Phase 2 history keys: {keys2}\")\n        \n        # Combine only common keys\n        common_keys = keys1.intersection(keys2) if history2 else keys1\n        \n        for key in common_keys:\n            try:\n                if history2:\n                    combined_history[key] = history1.history[key] + history2.history[key]\n                else:\n                    combined_history[key] = history1.history[key]\n            except Exception as e:\n                print(f\"Warning: Could not combine key '{key}': {e}\")\n                combined_history[key] = history1.history[key]\n        \n        # Handle missing keys by using alternative names\n        if 'accuracy' not in combined_history and 'acc' in combined_history:\n            combined_history['accuracy'] = combined_history['acc']\n        if 'val_accuracy' not in combined_history and 'val_acc' in combined_history:\n            combined_history['val_accuracy'] = combined_history['val_acc']\n            \n        return combined_history\n        \n    def train_model(self, model_name='resnet50v2', input_size=224, epochs=30, batch_size=32, fine_tune=True):\n        \"\"\"Train the pretrained model\"\"\"\n        print(f\"Training {model_name} pretrained model...\")\n        \n        # Create model\n        self.model, base_model = self.create_pretrained_model(model_name, input_size)\n        \n        self.model.compile(\n            optimizer=Adam(learning_rate=0.001),\n            loss='categorical_crossentropy',\n            metrics=['accuracy']  \n        )\n        \n        print(self.model.summary())\n        \n        # Setup callbacks\n        callbacks = [\n            EarlyStopping(\n                monitor='val_accuracy',  \n                patience=10,\n                restore_best_weights=True,\n                verbose=1\n            ),\n            ReduceLROnPlateau(\n                monitor='val_loss',\n                factor=0.5,\n                patience=5,\n                min_lr=1e-7,\n                verbose=1\n            ),\n            ModelCheckpoint(\n                f'best_{model_name}_model.h5',\n                monitor='val_accuracy',  \n                save_best_only=True,\n                verbose=1\n            )\n        ]\n        \n        # Setup data augmentation\n        datagen = self.setup_data_augmentation()\n        datagen.fit(self.X_train_split)\n        \n        # Initial training phase\n        print(\"Phase 1: Training with frozen base model...\")\n        initial_epochs = epochs // 2\n        \n        try:\n            history1 = self.model.fit(\n                datagen.flow(self.X_train_split, self.y_train_split, batch_size=batch_size),\n                epochs=initial_epochs,\n                validation_data=(self.X_val, self.y_val),\n                callbacks=callbacks,\n                verbose=1\n            )\n            \n            print(f\"Phase 1 completed. History keys: {list(history1.history.keys())}\")\n            \n        except Exception as e:\n            print(f\"Error in Phase 1: {e}\")\n            raise\n        \n        # Fine-tuning phase\n        history2 = None\n        if fine_tune:\n            try:\n                print(\"Phase 2: Fine-tuning...\")\n                self.fine_tune_model(base_model)\n                \n                # Continue training\n                history2 = self.model.fit(\n                    datagen.flow(self.X_train_split, self.y_train_split, batch_size=batch_size),\n                    epochs=epochs - initial_epochs,\n                    initial_epoch=initial_epochs,\n                    validation_data=(self.X_val, self.y_val),\n                    callbacks=callbacks,\n                    verbose=1\n                )\n                \n                print(f\"Phase 2 completed. History keys: {list(history2.history.keys())}\")\n                \n            except Exception as e:\n                print(f\"Error in Phase 2 (Fine-tuning): {e}\")\n                print(\"Continuing with Phase 1 results only...\")\n                fine_tune = False  # Disable fine_tune to use only phase1 results\n        \n        # Safely combine histories\n        self.history = self.safe_combine_histories(history1, history2)\n        \n        return self\n        \n    def evaluate_model(self):\n        \"\"\"Evaluate the model\"\"\"\n        print(\"Evaluating model...\")\n        \n        # Predictions on validation set\n        val_predictions = self.model.predict(self.X_val)\n        val_pred_classes = np.argmax(val_predictions, axis=1)\n        val_true_classes = np.argmax(self.y_val, axis=1)\n        \n        # Calculate accuracy\n        accuracy = accuracy_score(val_true_classes, val_pred_classes)\n        print(f\"Validation Accuracy: {accuracy:.4f}\")\n        \n        # Classification report\n        print(\"\\nClassification Report:\")\n        print(classification_report(val_true_classes, val_pred_classes))\n        \n        # Confusion matrix\n        cm = confusion_matrix(val_true_classes, val_pred_classes)\n        plt.figure(figsize=(10, 8))\n        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n        plt.title('Confusion Matrix')\n        plt.xlabel('Predicted')\n        plt.ylabel('Actual')\n        plt.show()\n        \n        return accuracy\n        \n    def plot_training_history(self):\n        \"\"\"Plot training history\"\"\"\n        if self.history is None:\n            print(\"No training history available\")\n            return\n            \n        # Check available keys\n        available_keys = list(self.history.keys())\n        print(f\"Available history keys: {available_keys}\")\n        \n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\n        \n        # Plot accuracy - handle different possible key names\n        acc_key = 'accuracy' if 'accuracy' in self.history else 'acc'\n        val_acc_key = 'val_accuracy' if 'val_accuracy' in self.history else 'val_acc'\n        \n        if acc_key in self.history:\n            ax1.plot(self.history[acc_key], label='Training Accuracy')\n        if val_acc_key in self.history:\n            ax1.plot(self.history[val_acc_key], label='Validation Accuracy')\n        ax1.set_title('Model Accuracy')\n        ax1.set_xlabel('Epoch')\n        ax1.set_ylabel('Accuracy')\n        ax1.legend()\n        \n        # Plot loss\n        if 'loss' in self.history:\n            ax2.plot(self.history['loss'], label='Training Loss')\n        if 'val_loss' in self.history:\n            ax2.plot(self.history['val_loss'], label='Validation Loss')\n        ax2.set_title('Model Loss')\n        ax2.set_xlabel('Epoch')\n        ax2.set_ylabel('Loss')\n        ax2.legend()\n        \n        plt.tight_layout()\n        plt.show()\n        \n    def make_predictions(self, save_path='pretrained_submission.csv'):\n        \"\"\"Make predictions on test set\"\"\"\n        print(\"Making predictions...\")\n        \n        # Predict on test set\n        test_predictions = self.model.predict(self.X_test)\n        test_pred_classes = np.argmax(test_predictions, axis=1)\n        \n        # Create submission file\n        submission = pd.DataFrame({\n            'ImageId': range(1, len(test_pred_classes) + 1),\n            'Label': test_pred_classes\n        })\n        \n        submission.to_csv(save_path, index=False)\n        print(f\"Submission saved to {save_path}\")\n        \n        return submission\n        \n    def visualize_predictions(self, num_samples=10):\n        \"\"\"Visualize some predictions\"\"\"\n        # Get some validation samples\n        sample_indices = np.random.choice(len(self.X_val), num_samples, replace=False)\n        sample_images = self.X_val[sample_indices]\n        sample_labels = self.y_val[sample_indices]\n        \n        # Make predictions\n        predictions = self.model.predict(sample_images)\n        predicted_classes = np.argmax(predictions, axis=1)\n        true_classes = np.argmax(sample_labels, axis=1)\n        \n        # Plot (show original 28x28 digit for better visualization)\n        fig, axes = plt.subplots(2, 5, figsize=(12, 6))\n        axes = axes.ravel()\n        \n        for i in range(num_samples):\n            # Convert back to grayscale for display\n            img_display = sample_images[i][:, :, 0]  # Take first channel\n            axes[i].imshow(img_display, cmap='gray')\n            axes[i].set_title(f'True: {true_classes[i]}, Pred: {predicted_classes[i]}')\n            axes[i].axis('off')\n            \n        plt.tight_layout()\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T06:04:04.204341Z","iopub.execute_input":"2025-07-22T06:04:04.204602Z","iopub.status.idle":"2025-07-22T06:04:04.237549Z","shell.execute_reply.started":"2025-07-22T06:04:04.204576Z","shell.execute_reply":"2025-07-22T06:04:04.236772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"recognizer = PretrainedDigitRecognizer()\n        \n# Load and preprocess data\nrecognizer.load_data('/kaggle/input/digit-recognizer/train.csv', '/kaggle/input/digit-recognizer/test.csv')\nrecognizer.preprocess_data(input_size=64)  # Standard size for most pretrained models\n\n# Train different models (choose one)\nmodels = ['resnet50v2', 'efficientnet', 'mobilenet']\n\nfor model_name in models:\n    print(f\"\\n{'='*50}\")\n    print(f\"Training {model_name.upper()}\")\n    print(f\"{'='*50}\")\n    \n    try:\n        # Train the model\n        recognizer.train_model(\n            model_name=model_name, \n            input_size=64, \n            epochs=20, \n            batch_size=32, \n            fine_tune=True\n        )\n        \n        # Evaluate the model\n        accuracy = recognizer.evaluate_model()\n        \n        # Plot training history\n        recognizer.plot_training_history()\n        \n        # Visualize some predictions\n        recognizer.visualize_predictions()\n        \n        # Make final predictions\n        submission = recognizer.make_predictions(f'{model_name}_submission.csv')\n        \n        print(f\"\\nFinal validation accuracy for {model_name}: {accuracy:.4f}\")\n        print(f\"Submission file created: {model_name}_submission.csv\")\n        \n    except Exception as e:\n        print(f\"Error training {model_name}: {str(e)}\")\n        continue","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T06:04:04.238367Z","iopub.execute_input":"2025-07-22T06:04:04.238620Z","iopub.status.idle":"2025-07-22T06:33:50.399479Z","shell.execute_reply.started":"2025-07-22T06:04:04.238597Z","shell.execute_reply":"2025-07-22T06:33:50.398705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}