{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# DSEG660 Challenge — CIFAR-10 Custom Residual CNN \n# AdamW + Warmup-Cosine LR + CutMix & CutOut + Label Smoothing\n\n\nimport os, random, math, glob\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, regularizers\nfrom tensorflow.keras.datasets import cifar10\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.optimizers import AdamW\n\nprint(\"TensorFlow version:\", tf.__version__)\n\n# --------------------------\n# Reproducibility\n# --------------------------\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED); tf.random.set_seed(SEED)\n\n# --------------------------\n# Kaggle paths (for later submission step)\n# --------------------------\nDATA_DIR = \"/kaggle/input/cifar10-object-recognition-in-images-zip-file\"\nTEST_DIR = os.path.join(DATA_DIR, \"train_test\")\nSAMPLE_SUB_PATH = os.path.join(DATA_DIR, \"sampleSubmission.csv\")\n\n# ============================================================\n# 1) Load CIFAR-10 dataset (official Keras) for TRAIN/VAL\n# ============================================================\n(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n\ndef normalize_img(x):\n    # normalize to [-1, 1]\n    return (x.astype(\"float32\") / 255.0 - 0.5) * 2.0\n\nx_train = normalize_img(x_train)\nx_test  = normalize_img(x_test)\ny_train = y_train.reshape(-1).astype(\"int32\")\ny_test  = y_test.reshape(-1).astype(\"int32\")\nprint(\"Train:\", x_train.shape, \" Test:\", x_test.shape)\n\n# 5% validation split\nn = x_train.shape[0]; n_val = int(0.05*n)\nx_val, y_val = x_train[:n_val], y_train[:n_val]\nx_tr , y_tr  = x_train[n_val:], y_train[n_val:]\n\n# ============================================================\n# 2) Model: Custom Residual CNN (modified ResNet-style)\n# ============================================================\ndef conv_block(x, filters, stride=1):\n    y = layers.Conv2D(filters, 3, strides=stride, padding='same',\n                      kernel_regularizer=regularizers.l2(1e-4), use_bias=False)(x)\n    y = layers.BatchNormalization()(y)\n    y = layers.Activation('relu')(y)\n    return y\n\ndef residual_block(x, filters, downsample=False):\n    stride = 2 if downsample else 1\n    y = conv_block(x, filters, stride)\n    y = conv_block(y, filters)\n    if downsample or x.shape[-1] != filters:\n        x = layers.Conv2D(filters, 1, strides=stride, padding='same',\n                          kernel_regularizer=regularizers.l2(1e-4), use_bias=False)(x)\n        x = layers.BatchNormalization()(x)\n    out = layers.add([x, y])\n    return layers.Activation('relu')(out)\n\ndef build_custom_resnet():\n    inputs = layers.Input(shape=(32, 32, 3))\n    x = conv_block(inputs, 64)\n\n    # Stage 1: 3 blocks of 64\n    x = residual_block(x, 64)\n    x = residual_block(x, 64)\n    x = residual_block(x, 64)\n\n    # Stage 2: 3 blocks of 128\n    x = residual_block(x, 128, downsample=True)\n    x = residual_block(x, 128)\n    x = residual_block(x, 128)\n\n    # Stage 3: 2 blocks of 256\n    x = residual_block(x, 256, downsample=True)\n    x = residual_block(x, 256)\n\n    # Stage 4: 2 blocks of 512\n    x = residual_block(x, 512, downsample=True)\n    x = residual_block(x, 512)\n\n    # Head: GAP + custom dense layers\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dense(256, activation='relu')(x)   # 🔹 extra dense layer\n    x = layers.Dropout(0.4)(x)                    # 🔹 extra dropout\n    outputs = layers.Dense(10, activation='softmax')(x)\n\n    return models.Model(inputs, outputs)\n\nmodel = build_custom_resnet()\nmodel.summary()\n\n# ============================================================\n# 3) Advanced Augmentations: CutMix + CutOut (tf.data)\n# ============================================================\nIMG_SIZE = 32\nNUM_CLASSES = 10\nCUTMIX_PROB = 0.7\nCUTOUT_PROB = 0.5\nCUTOUT_SIZE = 8\n\n@tf.function\ndef _beta(alpha=1.0):\n    g1 = tf.random.gamma([], alpha, 1.0)\n    g2 = tf.random.gamma([], alpha, 1.0)\n    return g1/(g1+g2+1e-7)\n\ndef cutmix(images, labels, alpha=1.0):\n    lam = _beta(alpha)\n    bs = tf.shape(images)[0]\n    idx = tf.random.shuffle(tf.range(bs))\n    x2, y2 = tf.gather(images, idx), tf.gather(labels, idx)\n\n    H, W = IMG_SIZE, IMG_SIZE\n    rx = tf.random.uniform([], 0, W, dtype=tf.int32)\n    ry = tf.random.uniform([], 0, H, dtype=tf.int32)\n    cut = tf.cast(tf.sqrt(1. - lam) * tf.cast(W, tf.float32), tf.int32)\n    x1 = tf.clip_by_value(rx - cut//2, 0, W); x2b = tf.clip_by_value(rx + cut//2, 0, W)\n    y1 = tf.clip_by_value(ry - cut//2, 0, H); y2b = tf.clip_by_value(ry + cut//2, 0, H)\n\n    mask = tf.ones((y2b-y1, x2b-x1, 3), images.dtype)\n    mask = tf.image.pad_to_bounding_box(mask, y1, x1, H, W)\n    mixed = images*(1.0 - mask) + x2*mask\n\n    lam_adj = 1. - tf.cast((x2b-x1)*(y2b-y1), tf.float32)/tf.cast(H*W, tf.float32)\n    y = lam_adj*tf.one_hot(labels, NUM_CLASSES) + (1. - lam_adj)*tf.one_hot(y2, NUM_CLASSES)\n    return mixed, y\n\ndef cutout(images, size=CUTOUT_SIZE):\n    def _one(img):\n        cx = tf.random.uniform([], 0, IMG_SIZE, dtype=tf.int32)\n        cy = tf.random.uniform([], 0, IMG_SIZE, dtype=tf.int32)\n        x1 = tf.clip_by_value(cx - size//2, 0, IMG_SIZE); x2 = tf.clip_by_value(cx + size//2, 0, IMG_SIZE)\n        y1 = tf.clip_by_value(cy - size//2, 0, IMG_SIZE); y2 = tf.clip_by_value(cy + size//2, 0, IMG_SIZE)\n        m = tf.ones((y2-y1, x2-x1, 3), img.dtype)\n        m = tf.image.pad_to_bounding_box(m, y1, x1, IMG_SIZE, IMG_SIZE)\n        return img*(1.0 - m)\n    return tf.map_fn(_one, images)\n\ndef augment_batch(images, labels):\n    images = tf.image.random_flip_left_right(images)\n    images = tf.image.resize_with_crop_or_pad(images, IMG_SIZE+4, IMG_SIZE+4)\n    images = tf.image.random_crop(images, [tf.shape(images)[0], IMG_SIZE, IMG_SIZE, 3])\n    do_cm = tf.less(tf.random.uniform([]), CUTMIX_PROB)\n    images, labels = tf.cond(do_cm,\n                             lambda: cutmix(images, labels),\n                             lambda: (images, tf.one_hot(labels, NUM_CLASSES)))\n    do_co = tf.less(tf.random.uniform([]), CUTOUT_PROB)\n    images = tf.cond(do_co, lambda: cutout(images), lambda: images)\n    return images, labels\n\ndef make_ds(xx, yy, train=True, batch=256, seed=SEED):\n    ds = tf.data.Dataset.from_tensor_slices((xx, yy))\n    if train:\n        ds = ds.shuffle(8192, seed=seed).batch(batch).map(augment_batch, num_parallel_calls=tf.data.AUTOTUNE)\n    else:\n        ds = ds.batch(batch).map(lambda a,b: (a, tf.one_hot(b, NUM_CLASSES)), num_parallel_calls=tf.data.AUTOTUNE)\n    return ds.prefetch(tf.data.AUTOTUNE)\n\n# Build datasets\nBATCH_SIZE = 256\ntrain_ds = make_ds(x_tr, y_tr, True,  batch=BATCH_SIZE)\nval_ds   = make_ds(x_val, y_val, False, batch=BATCH_SIZE)\n\n# ============================================================\n# 4) Optimizer + Warmup-Cosine LR + Label Smoothing (fixed)\n# ============================================================\nEPOCHS = 200\nWARMUP_EPOCHS = 5\nBASE_LR = 2e-3       # adjust if you change batch size\nMIN_LR  = 5e-6\nWEIGHT_DECAY = 1e-4\n\nsteps_per_epoch = math.ceil(x_tr.shape[0] / BATCH_SIZE)\n\nclass WarmupCosine(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, base_lr, warmup_epochs, total_epochs, steps_per_epoch, min_lr=0.0):\n        super().__init__()\n        self.base_lr = base_lr\n        self.warmup_steps = warmup_epochs * steps_per_epoch\n        self.total_steps  = total_epochs  * steps_per_epoch\n        self.min_lr = min_lr\n\n    def __call__(self, step):\n        step = tf.cast(step, tf.float32)\n        warm = self.base_lr * (step / tf.maximum(1., tf.cast(self.warmup_steps, tf.float32)))\n        prog = tf.clip_by_value((step - self.warmup_steps) /\n                                tf.maximum(1., self.total_steps - self.warmup_steps), 0., 1.)\n        cos  = self.min_lr + 0.5*(self.base_lr - self.min_lr)*(1. + tf.cos(math.pi*prog))\n        return tf.where(step < self.warmup_steps, warm, cos)\n\n    def get_config(self):  # 🔹 needed for TF 2.18+ serialization\n        return {\n            \"base_lr\": self.base_lr,\n            \"warmup_steps\": self.warmup_steps,\n            \"total_steps\": self.total_steps,\n            \"min_lr\": self.min_lr,\n        }\n\n# Instantiate schedule + optimizer\nlr_schedule = WarmupCosine(BASE_LR, WARMUP_EPOCHS, EPOCHS, steps_per_epoch, MIN_LR)\noptimizer   = AdamW(learning_rate=lr_schedule, weight_decay=WEIGHT_DECAY)\n\n# Compile model with label smoothing\nmodel.compile(\n    optimizer=optimizer,\n    loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n    metrics=[\"accuracy\"]\n)\n\n# ============================================================\n# 5) Train (checkpoint best val_accuracy)\n# ============================================================\nckpt = ModelCheckpoint(\"best.keras\", monitor=\"val_accuracy\", mode=\"max\",\n                       save_best_only=True, verbose=1)\nearly = EarlyStopping(monitor=\"val_accuracy\", mode=\"max\",\n                      patience=30, restore_best_weights=True, verbose=1)\n\nhistory = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=EPOCHS,\n    callbacks=[ckpt, early],\n    verbose=2\n)\n\n# Evaluate validation accuracy\nval_loss, val_acc = model.evaluate(val_ds, verbose=0)\nprint(f\"✅ Val accuracy (held-out 5%): {val_acc:.4f} | Val loss: {val_loss:.4f}\")\n\n# ============================================================\n# 6) Load BEST model for inference\n# ============================================================\nbest_model = tf.keras.models.load_model(\"best.keras\", compile=False)\nprint(\"✅ Loaded best checkpoint.\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:04:48.855194Z","iopub.execute_input":"2025-11-05T18:04:48.855855Z","iopub.status.idle":"2025-11-05T19:23:18.611185Z","shell.execute_reply.started":"2025-11-05T18:04:48.855828Z","shell.execute_reply":"2025-11-05T19:23:18.610397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === CIFAR-10 | Build /kaggle/working/submission.csv from best.keras ===\nimport os, glob, numpy as np, pandas as pd, tensorflow as tf, subprocess, shutil\n\n# --- Paths (competition mount) ---\nDATA_DIR   = \"/kaggle/input/cifar-10\"\nTEST_7Z    = os.path.join(DATA_DIR, \"test.7z\")\nWORK_DIR   = \"/kaggle/working\"\nEXTRACT_TO = os.path.join(WORK_DIR, \"test\")          # we will extract here\nMODEL_PATH = os.path.join(WORK_DIR, \"best.keras\")    # created by your training cell\nSAMPLE_SUB = os.path.join(DATA_DIR, \"sampleSubmission.csv\")\n\nprint(\"probe ok\")\nprint(\"DATA_DIR :\", DATA_DIR)\nprint(\"TEST_DIR  :\", EXTRACT_TO)\nprint(\"sampleSubmission exists:\", os.path.exists(SAMPLE_SUB))\n\n# --- Safety checks ---\nassert os.path.exists(MODEL_PATH), \"best.keras not found in /kaggle/working. Run the training cell first.\"\n\n# --- Ensure test images are extracted (handles both test/ and test/test layouts) ---\nif not os.path.exists(EXTRACT_TO):\n    os.makedirs(EXTRACT_TO, exist_ok=True)\n\n# If there are clearly no images yet, extract\nneed_extract = True\nfor pat in (os.path.join(EXTRACT_TO, \"*.png\"),\n            os.path.join(EXTRACT_TO, \"test\", \"*.png\")):\n    if glob.glob(pat):\n        need_extract = False\n        break\n\nif need_extract:\n    print(\"📦 Extracting test.7z ...\")\n    # -y overwrite, quiet the console noise\n    ret = os.system(f\"7z x '{TEST_7Z}' -o'{EXTRACT_TO}' -y > /dev/null\")\n    if ret != 0:\n        raise RuntimeError(\"7z extraction failed. Is test.7z attached under 'Add Input'?\")\n\n# Normalise path in case archive created test/test\nCANDIDATES = [EXTRACT_TO, os.path.join(EXTRACT_TO, \"test\")]\nTEST_DIR = next((p for p in CANDIDATES if os.path.exists(p)), EXTRACT_TO)\n\n# --- Discover images ---\npngs = sorted(glob.glob(os.path.join(TEST_DIR, \"**\", \"*.png\"), recursive=True))\nprint(f\"📸 Found {len(pngs)} images under {TEST_DIR}\")\nassert len(pngs) == 300000, f\"Expected 300000 test images, found {len(pngs)}\"\n\n# --- tf.data pipeline (resize to 32x32 and scale to [-1,1]) ---\ndef load_image(path):\n    img = tf.io.read_file(path)\n    img = tf.io.decode_image(img, channels=3, expand_animations=False)\n    img = tf.image.resize(img, [32, 32])\n    img = tf.image.convert_image_dtype(img, tf.float32)  # [0,1]\n    img = (img - 0.5) * 2.0                              # [-1,1]\n    return img\n\nBATCH = 1024\nds = (tf.data.Dataset.from_tensor_slices(pngs)\n        .map(load_image, num_parallel_calls=tf.data.AUTOTUNE)\n        .batch(BATCH)\n        .prefetch(tf.data.AUTOTUNE))\n\n# --- Load model & predict ---\nbest_model = tf.keras.models.load_model(MODEL_PATH, compile=False)\nprint(\"✅ Loaded best.keras\")\n\nprobs = best_model.predict(ds, verbose=1)\npred_idx = np.argmax(probs, axis=1)\n\n# --- Build submission (id comes from filename) ---\nlabel_names = [\"airplane\",\"automobile\",\"bird\",\"cat\",\"deer\",\"dog\",\"frog\",\"horse\",\"ship\",\"truck\"]\npred_labels = [label_names[i] for i in pred_idx]\nids = [int(os.path.splitext(os.path.basename(p))[0]) for p in pngs]\n\nsub = pd.DataFrame({\"id\": ids, \"label\": pred_labels}).sort_values(\"id\")\nassert len(sub) == 300000, f\"Submission length is {len(sub)} not 300000\"\n\n# --- Save into /kaggle/working and expose a duplicate name to refresh sidebar ---\nSAVE_PATH = os.path.join(WORK_DIR, \"submission.csv\")\nsub.to_csv(SAVE_PATH, index=False)\n\nEXPOSED = os.path.join(WORK_DIR, \"submission_rashid.csv\")  # duplicate helps sidebar refresh\ntry:\n    shutil.copy(SAVE_PATH, EXPOSED)\nexcept Exception:\n    pass\n\nprint(f\"\\n📁 Saved: {SAVE_PATH}\")\nprint(sub.head())\n\n# Show what's in /kaggle/working so you can click-download from the sidebar\nprint(\"\\n/kaggle/working contents:\")\nsubprocess.run([\"ls\",\"-lh\",\"/kaggle/working\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:27:34.543211Z","iopub.execute_input":"2025-11-05T19:27:34.543919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, subprocess, pandas as pd\nprint(\"Contents of /kaggle/working:\")\nsubprocess.run([\"ls\",\"-lh\",\"/kaggle/working\"])\n\np = \"/kaggle/working/submission.csv\"\nprint(\"\\nFound submission.csv?\", os.path.exists(p))\nif os.path.exists(p):\n    print(\"Size (bytes):\", os.path.getsize(p))\n    print(pd.read_csv(p, nrows=5))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}