{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":3004,"databundleVersionId":861823}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Enable both T4 GPUs\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs: {strategy.num_replicas_in_sync}\")\n\n# All models must be built inside strategy.scope()\nwith strategy.scope():\n    model = build_cnn(seed=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T14:24:27.092322Z","iopub.execute_input":"2026-03-23T14:24:27.092662Z","iopub.status.idle":"2026-03-23T14:24:27.100173Z","shell.execute_reply.started":"2026-03-23T14:24:27.092633Z","shell.execute_reply":"2026-03-23T14:24:27.098973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport shutil\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import (EarlyStopping,\n    LearningRateScheduler, ModelCheckpoint)\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(42)\ntf.random.set_seed(42)\n\n# ══════════════════════════════════════════════════════════════\n# ENABLE BOTH T4 GPUs\n# ══════════════════════════════════════════════════════════════\nstrategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of GPUs available: {strategy.num_replicas_in_sync}\")\n\n# Batch size must be multiplied by number of GPUs\nNUM_GPUS = strategy.num_replicas_in_sync\n\n# ══════════════════════════════════════════════════════════════\n# STEP 1 — RESTORE CHECKPOINTS\n# ══════════════════════════════════════════════════════════════\nprint(\"=\"*60)\nprint(\"STEP 1 — Restoring checkpoints...\")\nprint(\"=\"*60)\n\ncheckpoint_input = \"/kaggle/input/digit-recognizer-checkpoints\"\nif os.path.exists(checkpoint_input):\n    for f in os.listdir(checkpoint_input):\n        src = os.path.join(checkpoint_input, f)\n        dst = os.path.join(\"/kaggle/working\", f)\n        if not os.path.exists(dst):\n            shutil.copy(src, dst)\n            print(f\"Restored: {f}\")\n    print(\"All checkpoints restored!\")\nelse:\n    print(\"No previous checkpoints — starting fresh!\")\n\n# ══════════════════════════════════════════════════════════════\n# STEP 2 — LOAD & PREPROCESS DATA\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 2 — Loading data...\")\nprint(\"=\"*60)\n\ntrain = pd.read_csv(\n    \"/kaggle/input/competitions/digit-recognizer/train.csv\")\ntest  = pd.read_csv(\n    \"/kaggle/input/competitions/digit-recognizer/test.csv\")\n\ny      = train[\"label\"]\nX      = train.drop(\"label\",axis=1).values.astype(\"float32\")/255.0\nX_test = test.values.astype(\"float32\") / 255.0\n\nX      = X.reshape(-1, 28, 28, 1)\nX_test = X_test.reshape(-1, 28, 28, 1)\ny      = to_categorical(y, num_classes=10)\n\nprint(\"Loading external MNIST data...\")\n(X_mnist, y_mnist), _ = keras.datasets.mnist.load_data()\nX_mnist = X_mnist.astype(\"float32\") / 255.0\nX_mnist = X_mnist.reshape(-1, 28, 28, 1)\ny_mnist = to_categorical(y_mnist, 10)\n\nX_train_full = np.concatenate([X, X_mnist])\ny_train_full = np.concatenate([y, y_mnist])\nprint(f\"Total training samples: {X_train_full.shape[0]}\")\n\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.1, random_state=42\n)\n\n# ══════════════════════════════════════════════════════════════\n# STEP 3 — AUGMENTATION FUNCTIONS\n# ══════════════════════════════════════════════════════════════\ndef mixup_batch(X, y, alpha=0.2):\n    lam  = np.random.beta(alpha, alpha)\n    idx  = np.random.permutation(len(X))\n    Xm   = (lam*X + (1-lam)*X[idx]).astype(\"float32\")\n    ym   = (lam*y + (1-lam)*y[idx]).astype(\"float32\")\n    return Xm, ym\n\ndef cutout(X, n_holes=1, length=8):\n    X = X.copy()\n    for img in X:\n        for _ in range(n_holes):\n            yc = np.random.randint(28)\n            xc = np.random.randint(28)\n            y1 = max(0, yc-length//2)\n            y2 = min(28, yc+length//2)\n            x1 = max(0, xc-length//2)\n            x2 = min(28, xc+length//2)\n            img[y1:y2, x1:x2, :] = 0\n    return X\n\ndef augmented_generator(X, y, batch_size=64):\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    datagen = ImageDataGenerator(\n        rotation_range=12,\n        zoom_range=0.12,\n        width_shift_range=0.12,\n        height_shift_range=0.12,\n        shear_range=0.1\n    )\n    gen = datagen.flow(X, y, batch_size=batch_size, shuffle=True)\n    while True:\n        Xb, yb = next(gen)\n        Xb     = cutout(Xb)\n        Xb, yb = mixup_batch(Xb, yb)\n        yield Xb, yb\n\ndef predict_tta(model, X_test, n_tta=25):\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    tta   = ImageDataGenerator(\n        rotation_range=8,\n        zoom_range=0.08,\n        width_shift_range=0.08,\n        height_shift_range=0.08\n    )\n    preds = model.predict(X_test, batch_size=128, verbose=0)\n    for _ in range(n_tta-1):\n        aug    = next(tta.flow(\n            X_test, batch_size=len(X_test), shuffle=False))\n        preds += model.predict(aug, batch_size=128, verbose=0)\n    return preds / n_tta\n\ndef cosine_schedule(epoch, lr):\n    return 1e-7 + 0.5*(0.001-1e-7)*(\n        1+np.cos(np.pi*epoch/60))\n\ndef get_callbacks(name, i):\n    return [\n        LearningRateScheduler(cosine_schedule, verbose=0),\n        EarlyStopping(\n            monitor=\"val_accuracy\",\n            patience=12,\n            restore_best_weights=True,\n            verbose=1\n        ),\n        ModelCheckpoint(\n            f\"/kaggle/working/snap_{name}_{i}_\"\n            \"{epoch:02d}.keras\",\n            save_freq=\"epoch\",\n            save_best_only=False,\n            verbose=0\n        )\n    ]\n\ndef train_model(model, X_tr, y_tr,\n                batch_size=64, name=\"m\", i=0):\n    # Multiply batch size by number of GPUs\n    batch_size = batch_size * NUM_GPUS\n    steps = len(X_tr) // batch_size\n    gen   = augmented_generator(X_tr, y_tr, batch_size)\n    model.fit(\n        gen,\n        steps_per_epoch=steps,\n        epochs=60,\n        validation_data=(X_val, y_val),\n        callbacks=get_callbacks(name, i),\n        verbose=1\n    )\n    return model\n\n# ══════════════════════════════════════════════════════════════\n# STEP 4 — MODEL ARCHITECTURES (inside strategy.scope)\n# ══════════════════════════════════════════════════════════════\ndef build_cnn(seed=42):\n    tf.random.set_seed(seed)\n    with strategy.scope():\n        inp = keras.Input(shape=(28,28,1))\n        x   = layers.Conv2D(32,(3,3),activation=\"relu\",\n                            padding=\"same\")(inp)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.Conv2D(32,(3,3),activation=\"relu\",\n                            padding=\"same\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.MaxPooling2D(2,2)(x)\n        x   = layers.Dropout(0.25)(x)\n        x   = layers.Conv2D(64,(3,3),activation=\"relu\",\n                            padding=\"same\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.Conv2D(64,(3,3),activation=\"relu\",\n                            padding=\"same\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.MaxPooling2D(2,2)(x)\n        x   = layers.Dropout(0.25)(x)\n        x   = layers.Conv2D(128,(3,3),activation=\"relu\",\n                            padding=\"same\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.Conv2D(128,(3,3),activation=\"relu\",\n                            padding=\"same\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.Dropout(0.25)(x)\n        x   = layers.Flatten()(x)\n        x   = layers.Dense(512,activation=\"relu\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.Dropout(0.4)(x)\n        x   = layers.Dense(256,activation=\"relu\")(x)\n        x   = layers.BatchNormalization()(x)\n        x   = layers.Dropout(0.3)(x)\n        out = layers.Dense(10,activation=\"softmax\")(x)\n        m   = keras.Model(inp, out)\n        m.compile(\n            optimizer=keras.optimizers.Adam(0.001),\n            loss=keras.losses.CategoricalCrossentropy(\n                label_smoothing=0.1),\n            metrics=[\"accuracy\"])\n    return m\n\ndef build_effB0(seed=42):\n    tf.random.set_seed(seed)\n    with strategy.scope():\n        inp  = keras.Input(shape=(28,28,1))\n        x    = layers.Resizing(32,32)(inp)\n        x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n        base = keras.applications.EfficientNetB0(\n            include_top=False,weights=None,input_tensor=x)\n        out  = layers.GlobalAveragePooling2D()(base.output)\n        out  = layers.BatchNormalization()(out)\n        out  = layers.Dropout(0.4)(out)\n        out  = layers.Dense(256,activation=\"relu\")(out)\n        out  = layers.Dropout(0.3)(out)\n        out  = layers.Dense(10,activation=\"softmax\")(out)\n        m    = keras.Model(inp,out)\n        m.compile(\n            optimizer=keras.optimizers.Adam(0.001),\n            loss=keras.losses.CategoricalCrossentropy(\n                label_smoothing=0.1),\n            metrics=[\"accuracy\"])\n    return m\n\ndef build_effB3(seed=42):\n    tf.random.set_seed(seed)\n    with strategy.scope():\n        inp  = keras.Input(shape=(28,28,1))\n        x    = layers.Resizing(48,48)(inp)\n        x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n        base = keras.applications.EfficientNetB3(\n            include_top=False,weights=None,input_tensor=x)\n        out  = layers.GlobalAveragePooling2D()(base.output)\n        out  = layers.BatchNormalization()(out)\n        out  = layers.Dropout(0.4)(out)\n        out  = layers.Dense(256,activation=\"relu\")(out)\n        out  = layers.Dropout(0.3)(out)\n        out  = layers.Dense(10,activation=\"softmax\")(out)\n        m    = keras.Model(inp,out)\n        m.compile(\n            optimizer=keras.optimizers.Adam(0.001),\n            loss=keras.losses.CategoricalCrossentropy(\n                label_smoothing=0.1),\n            metrics=[\"accuracy\"])\n    return m\n\ndef build_effB7(seed=42):\n    tf.random.set_seed(seed)\n    with strategy.scope():\n        inp  = keras.Input(shape=(28,28,1))\n        x    = layers.Resizing(96,96)(inp)\n        x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n        base = keras.applications.EfficientNetB7(\n            include_top=False,weights=None,input_tensor=x)\n        out  = layers.GlobalAveragePooling2D()(base.output)\n        out  = layers.BatchNormalization()(out)\n        out  = layers.Dropout(0.5)(out)\n        out  = layers.Dense(512,activation=\"relu\")(out)\n        out  = layers.Dropout(0.4)(out)\n        out  = layers.Dense(10,activation=\"softmax\")(out)\n        m    = keras.Model(inp,out)\n        m.compile(\n            optimizer=keras.optimizers.Adam(0.001),\n            loss=keras.losses.CategoricalCrossentropy(\n                label_smoothing=0.1),\n            metrics=[\"accuracy\"])\n    return m\n\ndef build_densenet(seed=42):\n    tf.random.set_seed(seed)\n    with strategy.scope():\n        inp  = keras.Input(shape=(28,28,1))\n        x    = layers.Resizing(32,32)(inp)\n        x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n        base = keras.applications.DenseNet121(\n            include_top=False,weights=None,input_tensor=x)\n        out  = layers.GlobalAveragePooling2D()(base.output)\n        out  = layers.BatchNormalization()(out)\n        out  = layers.Dropout(0.4)(out)\n        out  = layers.Dense(256,activation=\"relu\")(out)\n        out  = layers.Dropout(0.3)(out)\n        out  = layers.Dense(10,activation=\"softmax\")(out)\n        m    = keras.Model(inp,out)\n        m.compile(\n            optimizer=keras.optimizers.Adam(0.001),\n            loss=keras.losses.CategoricalCrossentropy(\n                label_smoothing=0.1),\n            metrics=[\"accuracy\"])\n    return m\n\ndef build_resnet(seed=42):\n    tf.random.set_seed(seed)\n    with strategy.scope():\n        inp  = keras.Input(shape=(28,28,1))\n        x    = layers.Resizing(56,56)(inp)\n        x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n        base = keras.applications.ResNet50V2(\n            include_top=False,weights=None,input_tensor=x)\n        out  = layers.GlobalAveragePooling2D()(base.output)\n        out  = layers.BatchNormalization()(out)\n        out  = layers.Dropout(0.4)(out)\n        out  = layers.Dense(256,activation=\"relu\")(out)\n        out  = layers.Dropout(0.3)(out)\n        out  = layers.Dense(10,activation=\"softmax\")(out)\n        m    = keras.Model(inp,out)\n        m.compile(\n            optimizer=keras.optimizers.Adam(0.001),\n            loss=keras.losses.CategoricalCrossentropy(\n                label_smoothing=0.1),\n            metrics=[\"accuracy\"])\n    return m\n\nconfigs = [\n    (\"cnn\",   build_cnn,      8, 64),\n    (\"effB0\", build_effB0,    7, 64),\n    (\"effB3\", build_effB3,    5, 32),\n    (\"effB7\", build_effB7,    3, 16),\n    (\"dense\", build_densenet, 6, 32),\n    (\"resnet\",build_resnet,   6, 32),\n]\n\n# ══════════════════════════════════════════════════════════════\n# STEP 5 — PHASE 1: Train 35 models\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 5 — PHASE 1: Training 35 models on 102k samples\")\nprint(\"=\"*60)\n\nall_preds = []\n\nfor name, builder, count, bsize in configs:\n    print(f\"\\n===== {name.upper()} ({count} models) =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p1_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"{name} {i+1}/{count} loaded ✅\")\n            all_preds.append(np.load(sp))\n            continue\n        print(f\"\\n--- {name} {i+1}/{count} ---\")\n        model = builder(seed=i*77)\n        model = train_model(\n            model,X_train_full,y_train_full,bsize,name,i)\n        snap_preds = []\n        for ep in [30,40,50,60]:\n            sp2 = (f\"/kaggle/working/snap_{name}\"\n                   f\"_{i}_{ep:02d}.keras\")\n            if os.path.exists(sp2):\n                m2 = keras.models.load_model(sp2)\n                snap_preds.append(\n                    predict_tta(m2,X_test,n_tta=25))\n                del m2\n        preds = (np.mean(snap_preds,axis=0)\n                 if snap_preds\n                 else predict_tta(model,X_test,n_tta=25))\n        np.save(sp, preds)\n        all_preds.append(preds)\n        print(f\"{name} {i+1} saved! ✅\")\n        del model\n        tf.keras.backend.clear_session()\n\n# ══════════════════════════════════════════════════════════════\n# STEP 6 — PHASE 2: Pseudo Labels Round 1\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 6 — Pseudo Labeling Round 1\")\nprint(\"=\"*60)\n\np1_avg        = np.mean(all_preds,axis=0)\npseudo_labels = np.argmax(p1_avg,axis=1)\nconfidence    = np.max(p1_avg,axis=1)\nmask1         = confidence > 0.999\nX_ps1         = X_test[mask1]\ny_ps1         = to_categorical(pseudo_labels[mask1],10)\nprint(f\"Pseudo samples round 1: {mask1.sum()}\")\nX_tr1         = np.concatenate([X_train_full,X_ps1])\ny_tr1         = np.concatenate([y_train_full,y_ps1])\nprint(f\"New training size: {X_tr1.shape[0]}\")\n\n# ══════════════════════════════════════════════════════════════\n# STEP 7 — PHASE 3: Retrain on pseudo round 1\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 7 — PHASE 3: 35 models on pseudo round 1\")\nprint(\"=\"*60)\n\npseudo_preds1 = []\n\nfor name, builder, count, bsize in configs:\n    print(f\"\\n===== {name.upper()} pseudo1 =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p3_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"Pseudo1 {name} {i+1} loaded ✅\")\n            pseudo_preds1.append(np.load(sp))\n            continue\n        print(f\"\\n--- Pseudo1 {name} {i+1}/{count} ---\")\n        model = builder(seed=i*99+1)\n        model = train_model(\n            model,X_tr1,y_tr1,bsize,f\"p3_{name}\",i)\n        preds = predict_tta(model,X_test,n_tta=25)\n        np.save(sp,preds)\n        pseudo_preds1.append(preds)\n        print(f\"Pseudo1 {name} {i+1} saved! ✅\")\n        del model\n        tf.keras.backend.clear_session()\n\n# ══════════════════════════════════════════════════════════════\n# STEP 8 — PHASE 4: Pseudo Labels Round 2\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 8 — Pseudo Labeling Round 2\")\nprint(\"=\"*60)\n\ncombined       = all_preds + pseudo_preds1\np3_avg         = np.mean(combined,axis=0)\npseudo_labels2 = np.argmax(p3_avg,axis=1)\nconfidence2    = np.max(p3_avg,axis=1)\nmask2          = confidence2 > 0.995\nX_ps2          = X_test[mask2]\ny_ps2          = to_categorical(pseudo_labels2[mask2],10)\nprint(f\"Pseudo samples round 2: {mask2.sum()}\")\nX_tr2          = np.concatenate([X_train_full,X_ps2])\ny_tr2          = np.concatenate([y_train_full,y_ps2])\nprint(f\"New training size: {X_tr2.shape[0]}\")\n\n# ══════════════════════════════════════════════════════════════\n# STEP 9 — PHASE 5: Retrain on pseudo round 2\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 9 — PHASE 5: 35 models on pseudo round 2\")\nprint(\"=\"*60)\n\npseudo_preds2 = []\n\nfor name, builder, count, bsize in configs:\n    print(f\"\\n===== {name.upper()} pseudo2 =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p5_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"Pseudo2 {name} {i+1} loaded ✅\")\n            pseudo_preds2.append(np.load(sp))\n            continue\n        print(f\"\\n--- Pseudo2 {name} {i+1}/{count} ---\")\n        model = builder(seed=i*55+2)\n        model = train_model(\n            model,X_tr2,y_tr2,bsize,f\"p5_{name}\",i)\n        preds = predict_tta(model,X_test,n_tta=25)\n        np.save(sp,preds)\n        pseudo_preds2.append(preds)\n        print(f\"Pseudo2 {name} {i+1} saved! ✅\")\n        del model\n        tf.keras.backend.clear_session()\n\n# ══════════════════════════════════════════════════════════════\n# STEP 10 — FINAL ENSEMBLE & SUBMISSION\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 10 — Final Ensemble & Submission\")\nprint(\"=\"*60)\n\nall_combined     = all_preds + pseudo_preds1 + pseudo_preds2\nfinal_preds      = np.mean(all_combined,axis=0)\npredicted_labels = np.argmax(final_preds,axis=1)\n\nprint(f\"Total models ensembled: {len(all_combined)}\")\n\nsubmission = pd.DataFrame({\n    \"ImageId\": range(1,len(predicted_labels)+1),\n    \"Label\":   predicted_labels\n})\nsubmission.to_csv(\"/kaggle/working/submission.csv\",index=False)\n\nprint(\"\\n\"+\"=\"*60)\nprint(\"ALL DONE! 🏆\")\nprint(\"=\"*60)\nprint(submission.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T14:24:30.786485Z","iopub.execute_input":"2026-03-23T14:24:30.787258Z","iopub.status.idle":"2026-03-23T14:24:48.746096Z","shell.execute_reply.started":"2026-03-23T14:24:30.787192Z","shell.execute_reply":"2026-03-23T14:24:48.744586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Check all saved checkpoints\nsaved = []\nfor f in os.listdir(\"/kaggle/working\"):\n    if f.endswith(\".npy\"):\n        saved.append(f)\n        \nprint(f\"Total checkpoints saved: {len(saved)}\")\nfor f in sorted(saved):\n    print(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T14:25:00.48Z","iopub.execute_input":"2026-03-23T14:25:00.480396Z","iopub.status.idle":"2026-03-23T14:25:00.486814Z","shell.execute_reply.started":"2026-03-23T14:25:00.480367Z","shell.execute_reply":"2026-03-23T14:25:00.486061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport shutil\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import (EarlyStopping,\n    LearningRateScheduler, ModelCheckpoint)\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(42)\ntf.random.set_seed(42)\n\n# ── Single GPU — more stable with EfficientNet ────────────────\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\nprint(\"Using GPU 0 — stable mode!\")\n\n# ── Restore checkpoints ───────────────────────────────────────\nprint(\"=\"*60)\nprint(\"STEP 1 — Restoring checkpoints...\")\nprint(\"=\"*60)\ncheckpoint_input = \"/kaggle/input/digit-recognizer-checkpoints\"\nif os.path.exists(checkpoint_input):\n    for f in os.listdir(checkpoint_input):\n        src = os.path.join(checkpoint_input, f)\n        dst = os.path.join(\"/kaggle/working\", f)\n        if not os.path.exists(dst):\n            shutil.copy(src, dst)\n            print(f\"Restored: {f}\")\n    print(\"Checkpoints restored!\")\nelse:\n    print(\"Starting fresh!\")\n\n# ── Load data ─────────────────────────────────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 2 — Loading data...\")\nprint(\"=\"*60)\ntrain = pd.read_csv(\n    \"/kaggle/input/competitions/digit-recognizer/train.csv\")\ntest  = pd.read_csv(\n    \"/kaggle/input/competitions/digit-recognizer/test.csv\")\n\ny      = train[\"label\"]\nX      = train.drop(\"label\",axis=1).values.astype(\"float32\")/255.0\nX_test = test.values.astype(\"float32\")/255.0\nX      = X.reshape(-1,28,28,1)\nX_test = X_test.reshape(-1,28,28,1)\ny      = to_categorical(y,num_classes=10)\n\nprint(\"Loading external MNIST...\")\n(X_mnist,y_mnist),_ = keras.datasets.mnist.load_data()\nX_mnist = X_mnist.astype(\"float32\")/255.0\nX_mnist = X_mnist.reshape(-1,28,28,1)\ny_mnist = to_categorical(y_mnist,10)\n\nX_train_full = np.concatenate([X,X_mnist])\ny_train_full = np.concatenate([y,y_mnist])\nprint(f\"Total training samples: {X_train_full.shape[0]}\")\n\nX_train,X_val,y_train,y_val = train_test_split(\n    X,y,test_size=0.1,random_state=42)\n\n# ── Augmentation ──────────────────────────────────────────────\ndef mixup_batch(X,y,alpha=0.2):\n    lam = np.random.beta(alpha,alpha)\n    idx = np.random.permutation(len(X))\n    return (lam*X+(1-lam)*X[idx]).astype(\"float32\"), \\\n           (lam*y+(1-lam)*y[idx]).astype(\"float32\")\n\ndef cutout(X,n_holes=1,length=8):\n    X = X.copy()\n    for img in X:\n        for _ in range(n_holes):\n            yc = np.random.randint(28)\n            xc = np.random.randint(28)\n            y1,y2 = max(0,yc-length//2),min(28,yc+length//2)\n            x1,x2 = max(0,xc-length//2),min(28,xc+length//2)\n            img[y1:y2,x1:x2,:] = 0\n    return X\n\ndef augmented_generator(X,y,batch_size=64):\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    datagen = ImageDataGenerator(\n        rotation_range=12, zoom_range=0.12,\n        width_shift_range=0.12, height_shift_range=0.12,\n        shear_range=0.1)\n    gen = datagen.flow(X,y,batch_size=batch_size,shuffle=True)\n    while True:\n        Xb,yb = next(gen)\n        Xb    = cutout(Xb)\n        Xb,yb = mixup_batch(Xb,yb)\n        yield Xb,yb\n\ndef predict_tta(model,X_test,n_tta=25):\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    tta = ImageDataGenerator(\n        rotation_range=8, zoom_range=0.08,\n        width_shift_range=0.08, height_shift_range=0.08)\n    preds = model.predict(X_test,batch_size=64,verbose=0)\n    for _ in range(n_tta-1):\n        aug    = next(tta.flow(X_test,\n                      batch_size=len(X_test),shuffle=False))\n        preds += model.predict(aug,batch_size=64,verbose=0)\n    return preds/n_tta\n\ndef cosine_schedule(epoch,lr):\n    return 1e-7+0.5*(0.001-1e-7)*(1+np.cos(np.pi*epoch/60))\n\ndef get_callbacks(name,i):\n    return [\n        LearningRateScheduler(cosine_schedule,verbose=0),\n        EarlyStopping(monitor=\"val_accuracy\",patience=12,\n                      restore_best_weights=True,verbose=1),\n        ModelCheckpoint(\n            f\"/kaggle/working/snap_{name}_{i}_{{epoch:02d}}.keras\",\n            save_freq=\"epoch\",save_best_only=False,verbose=0)\n    ]\n\ndef train_model(model,X_tr,y_tr,batch_size=64,name=\"m\",i=0):\n    steps = len(X_tr)//batch_size\n    gen   = augmented_generator(X_tr,y_tr,batch_size)\n    model.fit(gen,steps_per_epoch=steps,epochs=60,\n              validation_data=(X_val,y_val),\n              callbacks=get_callbacks(name,i),verbose=1)\n    return model\n\n# ── Model architectures (NO strategy.scope) ───────────────────\ndef build_cnn(seed=42):\n    tf.random.set_seed(seed)\n    inp = keras.Input(shape=(28,28,1))\n    x   = layers.Conv2D(32,(3,3),activation=\"relu\",padding=\"same\")(inp)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Conv2D(32,(3,3),activation=\"relu\",padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.MaxPooling2D(2,2)(x)\n    x   = layers.Dropout(0.25)(x)\n    x   = layers.Conv2D(64,(3,3),activation=\"relu\",padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Conv2D(64,(3,3),activation=\"relu\",padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.MaxPooling2D(2,2)(x)\n    x   = layers.Dropout(0.25)(x)\n    x   = layers.Conv2D(128,(3,3),activation=\"relu\",padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Conv2D(128,(3,3),activation=\"relu\",padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Dropout(0.25)(x)\n    x   = layers.Flatten()(x)\n    x   = layers.Dense(512,activation=\"relu\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Dropout(0.4)(x)\n    x   = layers.Dense(256,activation=\"relu\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Dropout(0.3)(x)\n    out = layers.Dense(10,activation=\"softmax\")(x)\n    m   = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_effB0(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(32,32)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.EfficientNetB0(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_effB3(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(48,48)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.EfficientNetB3(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_effB7(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(96,96)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.EfficientNetB7(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.5)(out)\n    out  = layers.Dense(512,activation=\"relu\")(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_densenet(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(32,32)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.DenseNet121(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_resnet(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(56,56)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.ResNet50V2(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\nconfigs = [\n    (\"cnn\",   build_cnn,      8, 64),\n    (\"effB0\", build_effB0,    7, 64),\n    (\"effB3\", build_effB3,    5, 32),\n    (\"effB7\", build_effB7,    3, 16),\n    (\"dense\", build_densenet, 6, 32),\n    (\"resnet\",build_resnet,   6, 32),\n]\n\n# ── Phase 1 ───────────────────────────────────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 1 — Training 35 models\")\nprint(\"=\"*60)\nall_preds = []\n\nfor name,builder,count,bsize in configs:\n    print(f\"\\n===== {name.upper()} ({count} models) =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p1_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"{name} {i+1}/{count} loaded ✅\")\n            all_preds.append(np.load(sp))\n            continue\n        print(f\"\\n--- {name} {i+1}/{count} ---\")\n        model = builder(seed=i*77)\n        model = train_model(model,X_train_full,\n                            y_train_full,bsize,name,i)\n        snap_preds = []\n        for ep in [30,40,50,60]:\n            sp2 = (f\"/kaggle/working/snap_{name}\"\n                   f\"_{i}_{ep:02d}.keras\")\n            if os.path.exists(sp2):\n                m2 = keras.models.load_model(sp2)\n                snap_preds.append(\n                    predict_tta(m2,X_test,n_tta=25))\n                del m2\n        preds = (np.mean(snap_preds,axis=0)\n                 if snap_preds\n                 else predict_tta(model,X_test,n_tta=25))\n        np.save(sp,preds)\n        all_preds.append(preds)\n        print(f\"{name} {i+1} saved! ✅\")\n        del model\n        tf.keras.backend.clear_session()\n\n# ── Phase 2 — Pseudo Labels Round 1 ──────────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 2 — Pseudo Labeling Round 1\")\nprint(\"=\"*60)\np1_avg        = np.mean(all_preds,axis=0)\npseudo_labels = np.argmax(p1_avg,axis=1)\nconfidence    = np.max(p1_avg,axis=1)\nmask1         = confidence > 0.999\nX_ps1         = X_test[mask1]\ny_ps1         = to_categorical(pseudo_labels[mask1],10)\nprint(f\"Pseudo samples: {mask1.sum()}\")\nX_tr1         = np.concatenate([X_train_full,X_ps1])\ny_tr1         = np.concatenate([y_train_full,y_ps1])\nprint(f\"New training size: {X_tr1.shape[0]}\")\n\n# ── Phase 3 — Retrain on pseudo round 1 ──────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 3 — 35 models on pseudo round 1\")\nprint(\"=\"*60)\npseudo_preds1 = []\n\nfor name,builder,count,bsize in configs:\n    print(f\"\\n===== {name.upper()} pseudo1 =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p3_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"Pseudo1 {name} {i+1} loaded ✅\")\n            pseudo_preds1.append(np.load(sp))\n            continue\n        print(f\"\\n--- Pseudo1 {name} {i+1}/{count} ---\")\n        model = builder(seed=i*99+1)\n        model = train_model(model,X_tr1,y_tr1,\n                            bsize,f\"p3_{name}\",i)\n        preds = predict_tta(model,X_test,n_tta=25)\n        np.save(sp,preds)\n        pseudo_preds1.append(preds)\n        print(f\"Pseudo1 {name} {i+1} saved! ✅\")\n        del model\n        tf.keras.backend.clear_session()\n\n# ── Phase 4 — Pseudo Labels Round 2 ──────────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 4 — Pseudo Labeling Round 2\")\nprint(\"=\"*60)\ncombined       = all_preds+pseudo_preds1\np3_avg         = np.mean(combined,axis=0)\npseudo_labels2 = np.argmax(p3_avg,axis=1)\nconfidence2    = np.max(p3_avg,axis=1)\nmask2          = confidence2 > 0.995\nX_ps2          = X_test[mask2]\ny_ps2          = to_categorical(pseudo_labels2[mask2],10)\nprint(f\"Pseudo samples round 2: {mask2.sum()}\")\nX_tr2          = np.concatenate([X_train_full,X_ps2])\ny_tr2          = np.concatenate([y_train_full,y_ps2])\nprint(f\"New training size: {X_tr2.shape[0]}\")\n\n# ── Phase 5 — Retrain on pseudo round 2 ──────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 5 — 35 models on pseudo round 2\")\nprint(\"=\"*60)\npseudo_preds2 = []\n\nfor name,builder,count,bsize in configs:\n    print(f\"\\n===== {name.upper()} pseudo2 =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p5_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"Pseudo2 {name} {i+1} loaded ✅\")\n            pseudo_preds2.append(np.load(sp))\n            continue\n        print(f\"\\n--- Pseudo2 {name} {i+1}/{count} ---\")\n        model = builder(seed=i*55+2)\n        model = train_model(model,X_tr2,y_tr2,\n                            bsize,f\"p5_{name}\",i)\n        preds = predict_tta(model,X_test,n_tta=25)\n        np.save(sp,preds)\n        pseudo_preds2.append(preds)\n        print(f\"Pseudo2 {name} {i+1} saved! ✅\")\n        del model\n        tf.keras.backend.clear_session()\n\n# ── Final Ensemble ────────────────────────────────────────────\nprint(\"\\n\"+\"=\"*60)\nprint(\"FINAL ENSEMBLE — 105 models total\")\nprint(\"=\"*60)\nall_combined     = all_preds+pseudo_preds1+pseudo_preds2\nfinal_preds      = np.mean(all_combined,axis=0)\npredicted_labels = np.argmax(final_preds,axis=1)\nprint(f\"Total models: {len(all_combined)}\")\n\nsubmission = pd.DataFrame({\n    \"ImageId\": range(1,len(predicted_labels)+1),\n    \"Label\":   predicted_labels\n})\nsubmission.to_csv(\"/kaggle/working/submission.csv\",index=False)\nprint(\"\\nALL DONE! 🏆\")\nprint(submission.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T16:04:26.417312Z","iopub.execute_input":"2026-03-23T16:04:26.417945Z","iopub.status.idle":"2026-03-23T16:04:44.95076Z","shell.execute_reply.started":"2026-03-23T16:04:26.41791Z","shell.execute_reply":"2026-03-23T16:04:44.949119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport shutil\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import (EarlyStopping,\n    LearningRateScheduler)\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(42)\ntf.random.set_seed(42)\n\n# ══════════════════════════════════════════════════════════════\n# CLEAN DISK SPACE FIRST\n# ══════════════════════════════════════════════════════════════\nprint(\"=\"*60)\nprint(\"Cleaning disk space...\")\nprint(\"=\"*60)\ndeleted = 0\nfreed   = 0\nfor f in os.listdir(\"/kaggle/working\"):\n    if f.endswith(\".keras\") or f.endswith(\".h5\"):\n        path  = f\"/kaggle/working/{f}\"\n        size  = os.path.getsize(path)\n        os.remove(path)\n        freed   += size\n        deleted += 1\nprint(f\"Deleted {deleted} files\")\nprint(f\"Freed {freed/1024/1024/1024:.2f} GB\")\ntotal,used,free = shutil.disk_usage(\"/kaggle/working\")\nprint(f\"Free space now: {free/1024/1024/1024:.2f} GB\")\n\n# ══════════════════════════════════════════════════════════════\n# STEP 1 — RESTORE CHECKPOINTS\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 1 — Restoring checkpoints...\")\nprint(\"=\"*60)\ncheckpoint_input = \"/kaggle/input/digit-recognizer-checkpoints\"\nif os.path.exists(checkpoint_input):\n    for f in os.listdir(checkpoint_input):\n        if f.endswith(\".npy\"):\n            src = os.path.join(checkpoint_input, f)\n            dst = os.path.join(\"/kaggle/working\", f)\n            if not os.path.exists(dst):\n                shutil.copy(src, dst)\n                print(f\"Restored: {f}\")\n    print(\"Checkpoints restored!\")\nelse:\n    print(\"No previous checkpoints — starting fresh!\")\n\n# ══════════════════════════════════════════════════════════════\n# STEP 2 — LOAD DATA\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"STEP 2 — Loading data...\")\nprint(\"=\"*60)\ntrain = pd.read_csv(\n    \"/kaggle/input/competitions/digit-recognizer/train.csv\")\ntest  = pd.read_csv(\n    \"/kaggle/input/competitions/digit-recognizer/test.csv\")\n\ny      = train[\"label\"]\nX      = train.drop(\"label\",axis=1).values.astype(\"float32\")/255.0\nX_test = test.values.astype(\"float32\")/255.0\nX      = X.reshape(-1,28,28,1)\nX_test = X_test.reshape(-1,28,28,1)\ny      = to_categorical(y,num_classes=10)\n\nprint(\"Loading external MNIST...\")\n(X_mnist,y_mnist),_ = keras.datasets.mnist.load_data()\nX_mnist = X_mnist.astype(\"float32\")/255.0\nX_mnist = X_mnist.reshape(-1,28,28,1)\ny_mnist = to_categorical(y_mnist,10)\n\nX_train_full = np.concatenate([X,X_mnist])\ny_train_full = np.concatenate([y,y_mnist])\nprint(f\"Total training samples: {X_train_full.shape[0]}\")\n\nX_train,X_val,y_train,y_val = train_test_split(\n    X,y,test_size=0.1,random_state=42)\n\n# ══════════════════════════════════════════════════════════════\n# STEP 3 — AUGMENTATION\n# ══════════════════════════════════════════════════════════════\ndef mixup_batch(X,y,alpha=0.2):\n    lam = np.random.beta(alpha,alpha)\n    idx = np.random.permutation(len(X))\n    return (lam*X+(1-lam)*X[idx]).astype(\"float32\"), \\\n           (lam*y+(1-lam)*y[idx]).astype(\"float32\")\n\ndef cutout(X,n_holes=1,length=8):\n    X = X.copy()\n    for img in X:\n        for _ in range(n_holes):\n            yc = np.random.randint(28)\n            xc = np.random.randint(28)\n            y1,y2 = max(0,yc-length//2),min(28,yc+length//2)\n            x1,x2 = max(0,xc-length//2),min(28,xc+length//2)\n            img[y1:y2,x1:x2,:] = 0\n    return X\n\ndef augmented_generator(X,y,batch_size=64):\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    datagen = ImageDataGenerator(\n        rotation_range=12, zoom_range=0.12,\n        width_shift_range=0.12, height_shift_range=0.12,\n        shear_range=0.1)\n    gen = datagen.flow(X,y,batch_size=batch_size,shuffle=True)\n    while True:\n        Xb,yb = next(gen)\n        Xb    = cutout(Xb)\n        Xb,yb = mixup_batch(Xb,yb)\n        yield Xb,yb\n\ndef predict_tta(model,X_test,n_tta=25):\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    tta = ImageDataGenerator(\n        rotation_range=8, zoom_range=0.08,\n        width_shift_range=0.08, height_shift_range=0.08)\n    preds = model.predict(X_test,batch_size=64,verbose=0)\n    for _ in range(n_tta-1):\n        aug    = next(tta.flow(X_test,\n                      batch_size=len(X_test),shuffle=False))\n        preds += model.predict(aug,batch_size=64,verbose=0)\n    return preds/n_tta\n\ndef cosine_schedule(epoch,lr):\n    return 1e-7+0.5*(0.001-1e-7)*(1+np.cos(np.pi*epoch/60))\n\ndef get_callbacks():\n    return [\n        LearningRateScheduler(cosine_schedule,verbose=0),\n        EarlyStopping(monitor=\"val_accuracy\",patience=12,\n                      restore_best_weights=True,verbose=1)\n    ]\n\ndef train_and_save(model,X_tr,y_tr,batch_size,sp):\n    steps = len(X_tr)//batch_size\n    gen   = augmented_generator(X_tr,y_tr,batch_size)\n    model.fit(gen,steps_per_epoch=steps,epochs=60,\n              validation_data=(X_val,y_val),\n              callbacks=get_callbacks(),verbose=1)\n    preds = predict_tta(model,X_test,n_tta=25)\n    np.save(sp,preds)\n    del model\n    tf.keras.backend.clear_session()\n\n    # Check disk space after each model\n    _,_,free = shutil.disk_usage(\"/kaggle/working\")\n    print(f\"Free disk: {free/1024/1024/1024:.2f} GB\")\n    return preds\n\n# ══════════════════════════════════════════════════════════════\n# STEP 4 — MODEL ARCHITECTURES\n# ══════════════════════════════════════════════════════════════\ndef build_cnn(seed=42):\n    tf.random.set_seed(seed)\n    inp = keras.Input(shape=(28,28,1))\n    x   = layers.Conv2D(32,(3,3),activation=\"relu\",\n                        padding=\"same\")(inp)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Conv2D(32,(3,3),activation=\"relu\",\n                        padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.MaxPooling2D(2,2)(x)\n    x   = layers.Dropout(0.25)(x)\n    x   = layers.Conv2D(64,(3,3),activation=\"relu\",\n                        padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Conv2D(64,(3,3),activation=\"relu\",\n                        padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.MaxPooling2D(2,2)(x)\n    x   = layers.Dropout(0.25)(x)\n    x   = layers.Conv2D(128,(3,3),activation=\"relu\",\n                        padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Conv2D(128,(3,3),activation=\"relu\",\n                        padding=\"same\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Dropout(0.25)(x)\n    x   = layers.Flatten()(x)\n    x   = layers.Dense(512,activation=\"relu\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Dropout(0.4)(x)\n    x   = layers.Dense(256,activation=\"relu\")(x)\n    x   = layers.BatchNormalization()(x)\n    x   = layers.Dropout(0.3)(x)\n    out = layers.Dense(10,activation=\"softmax\")(x)\n    m   = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_effB0(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(32,32)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.EfficientNetB0(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_effB3(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(48,48)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.EfficientNetB3(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_effB7(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(96,96)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.EfficientNetB7(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.5)(out)\n    out  = layers.Dense(512,activation=\"relu\")(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_densenet(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(32,32)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.DenseNet121(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\ndef build_resnet(seed=42):\n    tf.random.set_seed(seed)\n    inp  = keras.Input(shape=(28,28,1))\n    x    = layers.Resizing(56,56)(inp)\n    x    = layers.Conv2D(3,(1,1),padding=\"same\")(x)\n    base = keras.applications.ResNet50V2(\n        include_top=False,weights=None,input_tensor=x)\n    out  = layers.GlobalAveragePooling2D()(base.output)\n    out  = layers.BatchNormalization()(out)\n    out  = layers.Dropout(0.4)(out)\n    out  = layers.Dense(256,activation=\"relu\")(out)\n    out  = layers.Dropout(0.3)(out)\n    out  = layers.Dense(10,activation=\"softmax\")(out)\n    m    = keras.Model(inp,out)\n    m.compile(optimizer=keras.optimizers.Adam(0.001),\n              loss=keras.losses.CategoricalCrossentropy(\n                  label_smoothing=0.1),\n              metrics=[\"accuracy\"])\n    return m\n\nconfigs = [\n    (\"cnn\",   build_cnn,      8, 64),\n    (\"effB0\", build_effB0,    7, 64),\n    (\"effB3\", build_effB3,    5, 32),\n    (\"effB7\", build_effB7,    3, 16),\n    (\"dense\", build_densenet, 6, 32),\n    (\"resnet\",build_resnet,   6, 32),\n]\n\n# ══════════════════════════════════════════════════════════════\n# PHASE 1 — Train 35 models\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 1 — Training 35 models on 102k samples\")\nprint(\"=\"*60)\nall_preds = []\n\nfor name,builder,count,bsize in configs:\n    print(f\"\\n===== {name.upper()} ({count} models) =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p1_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"{name} {i+1}/{count} loaded ✅\")\n            all_preds.append(np.load(sp))\n            continue\n        print(f\"\\n--- {name} {i+1}/{count} ---\")\n        model = builder(seed=i*77)\n        preds = train_and_save(model,X_train_full,\n                               y_train_full,bsize,sp)\n        all_preds.append(preds)\n        print(f\"{name} {i+1} saved! ✅\")\n\nprint(f\"\\nPhase 1 done! Models: {len(all_preds)}\")\n\n# ══════════════════════════════════════════════════════════════\n# PHASE 2 — Pseudo Labels Round 1\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 2 — Pseudo Labeling Round 1\")\nprint(\"=\"*60)\np1_avg        = np.mean(all_preds,axis=0)\npseudo_labels = np.argmax(p1_avg,axis=1)\nconfidence    = np.max(p1_avg,axis=1)\nmask1         = confidence > 0.999\nX_ps1         = X_test[mask1]\ny_ps1         = to_categorical(pseudo_labels[mask1],10)\nprint(f\"Pseudo samples added: {mask1.sum()}\")\nX_tr1         = np.concatenate([X_train_full,X_ps1])\ny_tr1         = np.concatenate([y_train_full,y_ps1])\nprint(f\"New training size: {X_tr1.shape[0]}\")\n\n# ══════════════════════════════════════════════════════════════\n# PHASE 3 — Retrain 35 models on pseudo round 1\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 3 — 35 models on pseudo round 1\")\nprint(\"=\"*60)\npseudo_preds1 = []\n\nfor name,builder,count,bsize in configs:\n    print(f\"\\n===== {name.upper()} pseudo1 =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p3_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"Pseudo1 {name} {i+1} loaded ✅\")\n            pseudo_preds1.append(np.load(sp))\n            continue\n        print(f\"\\n--- Pseudo1 {name} {i+1}/{count} ---\")\n        model = builder(seed=i*99+1)\n        preds = train_and_save(model,X_tr1,y_tr1,bsize,sp)\n        pseudo_preds1.append(preds)\n        print(f\"Pseudo1 {name} {i+1} saved! ✅\")\n\nprint(f\"\\nPhase 3 done! Models: {len(pseudo_preds1)}\")\n\n# ══════════════════════════════════════════════════════════════\n# PHASE 4 — Pseudo Labels Round 2\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 4 — Pseudo Labeling Round 2\")\nprint(\"=\"*60)\ncombined       = all_preds+pseudo_preds1\np3_avg         = np.mean(combined,axis=0)\npseudo_labels2 = np.argmax(p3_avg,axis=1)\nconfidence2    = np.max(p3_avg,axis=1)\nmask2          = confidence2 > 0.995\nX_ps2          = X_test[mask2]\ny_ps2          = to_categorical(pseudo_labels2[mask2],10)\nprint(f\"Pseudo samples round 2: {mask2.sum()}\")\nX_tr2          = np.concatenate([X_train_full,X_ps2])\ny_tr2          = np.concatenate([y_train_full,y_ps2])\nprint(f\"New training size: {X_tr2.shape[0]}\")\n\n# ══════════════════════════════════════════════════════════════\n# PHASE 5 — Retrain 35 models on pseudo round 2\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"PHASE 5 — 35 models on pseudo round 2\")\nprint(\"=\"*60)\npseudo_preds2 = []\n\nfor name,builder,count,bsize in configs:\n    print(f\"\\n===== {name.upper()} pseudo2 =====\")\n    for i in range(count):\n        sp = f\"/kaggle/working/p5_{name}_{i}.npy\"\n        if os.path.exists(sp):\n            print(f\"Pseudo2 {name} {i+1} loaded ✅\")\n            pseudo_preds2.append(np.load(sp))\n            continue\n        print(f\"\\n--- Pseudo2 {name} {i+1}/{count} ---\")\n        model = builder(seed=i*55+2)\n        preds = train_and_save(model,X_tr2,y_tr2,bsize,sp)\n        pseudo_preds2.append(preds)\n        print(f\"Pseudo2 {name} {i+1} saved! ✅\")\n\nprint(f\"\\nPhase 5 done! Models: {len(pseudo_preds2)}\")\n\n# ══════════════════════════════════════════════════════════════\n# FINAL ENSEMBLE\n# ══════════════════════════════════════════════════════════════\nprint(\"\\n\"+\"=\"*60)\nprint(\"FINAL ENSEMBLE — 105 models total\")\nprint(\"=\"*60)\nall_combined     = all_preds+pseudo_preds1+pseudo_preds2\nfinal_preds      = np.mean(all_combined,axis=0)\npredicted_labels = np.argmax(final_preds,axis=1)\nprint(f\"Total models ensembled: {len(all_combined)}\")\n\nsubmission = pd.DataFrame({\n    \"ImageId\": range(1,len(predicted_labels)+1),\n    \"Label\":   predicted_labels\n})\nsubmission.to_csv(\"/kaggle/working/submission.csv\",index=False)\nprint(\"\\nALL DONE! 🏆\")\nprint(submission.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T16:02:59.471299Z","iopub.execute_input":"2026-03-23T16:02:59.471825Z","iopub.status.idle":"2026-03-23T16:04:06.346949Z","shell.execute_reply.started":"2026-03-23T16:02:59.471794Z","shell.execute_reply":"2026-03-23T16:04:06.345726Z"}},"outputs":[],"execution_count":null}]}