{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nChallenge Assignment - CIFAR-10 Kaggle competition\nhttps://www.kaggle.com/c/cifar-10\n\nA custom neural network (\"DualPath-SE Net\") built specifically for this task.\n  - No pre-trained weights: every weight starts from random values\n  - Optimizer: AdamW (weight decay applied separately from the gradient step)\n  - Learning rate: linear warmup first, then cosine annealing\n  - Augmentation: CutMix and Cutout, plus random crop and horizontal flip\n\nKaggle setup: attach the CIFAR-10 competition data, pick the T4 GPU,\nand switch Internet on.\nExpected runtime: about 25 minutes.\n\"\"\"\n\nimport os\nimport glob\nimport shutil\nimport subprocess\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom keras import layers\n\nkeras.utils.set_random_seed(1234)\nkeras.mixed_precision.set_global_policy(\"mixed_float16\")\n\nprint(\"TensorFlow\", tf.__version__, \"| Keras\", keras.__version__)\nprint(\"GPUs:\", tf.config.list_physical_devices(\"GPU\"))\n\nCLASSES = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\",\n           \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\nAUTOTUNE = tf.data.AUTOTUNE\nBATCH = 512\nEPOCHS = 70\n\n# ---------------------------------------------------------------\n# 1. Loading the data\n# ---------------------------------------------------------------\ntry:\n    (x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\nexcept Exception as e:\n    raise RuntimeError(\"CIFAR-10 failed to download. Make sure Internet is \"\n                       \"turned on in the session options.\") from e\n\ny_train = y_train.ravel()\ny_test = y_test.ravel()\nprint(\"Train:\", x_train.shape, \"Test:\", x_test.shape)\n\n# ---------------------------------------------------------------\n# 2. Augmentation: crop + flip + Cutout per image, CutMix per batch\n# ---------------------------------------------------------------\n# Cutout erases a random square, so the model cannot lean on any one\n# patch of the picture. CutMix goes a step further: it pastes a square\n# taken from a different image and blends the two labels according to\n# how much area was pasted. This pushes the model to recognise objects\n# from partial views and keeps its confidence better calibrated.\n\n\ndef cutout(img, size=8):\n    cy = tf.random.uniform([], 0, 32, tf.int32)\n    cx = tf.random.uniform([], 0, 32, tf.int32)\n    yy = tf.range(32)[:, None]\n    xx = tf.range(32)[None, :]\n    inside = ((yy >= cy - size // 2) & (yy < cy + size // 2) &\n              (xx >= cx - size // 2) & (xx < cx + size // 2))\n    return img * tf.cast(tf.logical_not(inside), img.dtype)[..., None]\n\n\ndef per_image(img, label):\n    img = tf.pad(img, [[4, 4], [4, 4], [0, 0]], mode=\"REFLECT\")\n    img = tf.image.random_crop(img, [32, 32, 3])\n    img = tf.image.random_flip_left_right(img)\n    img = tf.cond(tf.random.uniform([]) < 0.5, lambda: cutout(img), lambda: img)\n    return tf.cast(img, tf.float32), tf.one_hot(label, 10)\n\n\ndef cutmix(images, labels):\n    lam = tf.random.uniform([], 0.0, 1.0)\n    half = tf.cast(tf.round(16.0 * tf.sqrt(1.0 - lam)), tf.int32)\n    cy = tf.random.uniform([], 0, 32, tf.int32)\n    cx = tf.random.uniform([], 0, 32, tf.int32)\n    y0 = tf.clip_by_value(cy - half, 0, 32)\n    y1 = tf.clip_by_value(cy + half, 0, 32)\n    x0 = tf.clip_by_value(cx - half, 0, 32)\n    x1 = tf.clip_by_value(cx + half, 0, 32)\n\n    yy = tf.range(32)[:, None]\n    xx = tf.range(32)[None, :]\n    inside = (yy >= y0) & (yy < y1) & (xx >= x0) & (xx < x1)\n    mask = tf.cast(inside, images.dtype)[None, :, :, None]\n\n    perm = tf.random.shuffle(tf.range(tf.shape(images)[0]))\n    mixed = images * (1.0 - mask) + tf.gather(images, perm) * mask\n\n    area = tf.cast((y1 - y0) * (x1 - x0), tf.float32) / (32.0 * 32.0)\n    keep = 1.0 - area\n    mixed_labels = labels * keep + tf.gather(labels, perm) * (1.0 - keep)\n    return mixed, mixed_labels\n\n\ndef maybe_cutmix(images, labels):\n    return tf.cond(tf.random.uniform([]) < 0.5,\n                   lambda: cutmix(images, labels),\n                   lambda: (images, labels))\n\n\ntrain_ds = (tf.data.Dataset.from_tensor_slices((x_train, y_train))\n            .shuffle(50000, reshuffle_each_iteration=True)\n            .map(per_image, num_parallel_calls=AUTOTUNE)\n            .batch(BATCH, drop_remainder=True)\n            .map(maybe_cutmix, num_parallel_calls=AUTOTUNE)\n            .prefetch(AUTOTUNE))\n\nval_ds = (tf.data.Dataset.from_tensor_slices((x_test, y_test))\n          .map(lambda im, lb: (tf.cast(im, tf.float32), tf.one_hot(lb, 10)))\n          .batch(1000)\n          .prefetch(AUTOTUNE))\n\n# ---------------------------------------------------------------\n# 3. The model: DualPath-SE Net\n# ---------------------------------------------------------------\n# This architecture is my own design rather than a standard ready-made\n# model. It is built around three ideas:\n#\n#   1. Dual-path block. Every block runs two 3x3 convolutions side by\n#      side: a regular one and a dilated one (rate 2). Together they\n#      cover a 3x3 and an effective 5x5 neighbourhood at the same time.\n#      Their outputs are joined and merged, which gives multi-scale\n#      vision (like Inception) while costing far less than a true 5x5\n#      convolution.\n#   2. Squeeze-and-excitation gate. The end of each block applies a\n#      channel-attention gate that rescales channels using global\n#      context, letting the network turn down channels that do not\n#      matter for the current image.\n#   3. Dual pooling head. Global average pooling and global max pooling\n#      are concatenated. The average summarises overall texture, while\n#      the max keeps the single strongest local response.\n#\n# Blocks are residual and use swish activations, which tend to train\n# more smoothly than ReLU at this depth.\n\nMEAN = np.array([0.4914, 0.4822, 0.4465])\nSTD = np.array([0.2470, 0.2435, 0.2616])\n\n\ndef se_gate(x, ch, ratio=8):\n    s = layers.GlobalAveragePooling2D(keepdims=True)(x)\n    s = layers.Conv2D(max(ch // ratio, 8), 1, activation=\"swish\")(s)\n    s = layers.Conv2D(ch, 1, activation=\"sigmoid\")(s)\n    return layers.Multiply()([x, s])\n\n\ndef dualpath_block(x, ch):\n    a = layers.Conv2D(ch // 2, 3, padding=\"same\", use_bias=False)(x)\n    b = layers.Conv2D(ch // 2, 3, padding=\"same\", dilation_rate=2,\n                      use_bias=False)(x)\n    y = layers.Concatenate()([a, b])\n    y = layers.BatchNormalization(momentum=0.9, epsilon=1e-5)(y)\n    y = layers.Activation(\"swish\")(y)\n\n    y = layers.Conv2D(ch, 3, padding=\"same\", use_bias=False)(y)\n    y = layers.BatchNormalization(momentum=0.9, epsilon=1e-5)(y)\n    y = se_gate(y, ch)\n\n    y = layers.Add()([x, y])\n    return layers.Activation(\"swish\")(y)\n\n\ndef transition(x, ch, pool=True):\n    x = layers.Conv2D(ch, 3, padding=\"same\", use_bias=False)(x)\n    x = layers.BatchNormalization(momentum=0.9, epsilon=1e-5)(x)\n    x = layers.Activation(\"swish\")(x)\n    if pool:\n        x = layers.MaxPooling2D(2)(x)\n    return x\n\n\ninputs = keras.Input(shape=(32, 32, 3))\nx = layers.Rescaling(1.0 / 255)(inputs)\nx = layers.Normalization(mean=MEAN, variance=STD ** 2)(x)\n\nx = transition(x, 64, pool=False)      # stem: 32x32x64\nx = dualpath_block(x, 64)\n\nx = transition(x, 128)                 # stage 2: 16x16x128\nx = dualpath_block(x, 128)\nx = dualpath_block(x, 128)\n\nx = transition(x, 256)                 # stage 3: 8x8x256\nx = dualpath_block(x, 256)\nx = dualpath_block(x, 256)\n\nx = transition(x, 512)                 # stage 4: 4x4x512\nx = dualpath_block(x, 512)\n\navg = layers.GlobalAveragePooling2D()(x)\nmx = layers.GlobalMaxPooling2D()(x)\nx = layers.Concatenate()([avg, mx])\nx = layers.Dropout(0.2)(x)\nx = layers.Dense(10)(x)\noutputs = layers.Activation(\"softmax\", dtype=\"float32\")(x)\n\nmodel = keras.Model(inputs, outputs, name=\"dualpath_se_net\")\nmodel.summary()\n\n# ---------------------------------------------------------------\n# 4. AdamW with warmup followed by cosine annealing\n# ---------------------------------------------------------------\n# Warmup keeps the first steps small while Adam's running estimates\n# are still unreliable. After that, the cosine curve lowers the\n# learning rate smoothly down to zero, which helps the model settle\n# into a flat, well-generalising minimum.\n\nsteps_per_epoch = len(x_train) // BATCH\nwarmup_steps = 5 * steps_per_epoch\ntotal_steps = EPOCHS * steps_per_epoch\n\nschedule = keras.optimizers.schedules.CosineDecay(\n    initial_learning_rate=0.0,\n    warmup_target=2.5e-3,\n    warmup_steps=warmup_steps,\n    decay_steps=total_steps - warmup_steps,\n    alpha=0.0,\n)\n\nmodel.compile(\n    optimizer=keras.optimizers.AdamW(learning_rate=schedule,\n                                     weight_decay=0.02),\n    loss=keras.losses.CategoricalCrossentropy(label_smoothing=0.1),\n    metrics=[\"accuracy\"],\n)\n\n# ---------------------------------------------------------------\n# 5. Training\n# ---------------------------------------------------------------\nt0 = time.time()\nhistory = model.fit(train_ds, validation_data=val_ds,\n                    epochs=EPOCHS, verbose=2)\ntrain_minutes = (time.time() - t0) / 60\nprint(f\"\\nTraining time: {train_minutes:.1f} minutes\")\n\n# ---------------------------------------------------------------\n# 6. Check accuracy on the official test set (a rough guide to the\n#    Kaggle score)\n# ---------------------------------------------------------------\n\n\ndef tta_predict(img_ds):\n    p = model.predict(img_ds, verbose=0)\n    p += model.predict(img_ds.map(tf.image.flip_left_right), verbose=0)\n    return p / 2\n\n\ntest_img_ds = (tf.data.Dataset.from_tensor_slices(x_test)\n               .map(lambda im: tf.cast(im, tf.float32))\n               .batch(1000))\n\ntest_probs = tta_predict(test_img_ds)\ntest_acc = (test_probs.argmax(axis=1) == y_test).mean()\nprint(f\"Official CIFAR-10 test accuracy (with TTA): {test_acc * 100:.2f}%\")\nprint(\"-> expected Kaggle score is about\", round(test_acc, 4))\n\n# ---------------------------------------------------------------\n# 7-9. Kaggle test set: decode the 300,000 PNGs directly from the\n#      archive into memory (extracting them to Kaggle's disk would take\n#      hours), predict, save submission.csv, and plot training curves\n# ---------------------------------------------------------------\nimport io, os, glob, subprocess, time\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\narchives = glob.glob(\"/kaggle/input/**/test.7z\", recursive=True)\nif not archives:\n    raise FileNotFoundError(\"test.7z not found. Attach the CIFAR-10 competition data \"\n                            \"via Add Input -> Competitions.\")\nARCHIVE = archives[0]\nCOMP_DIR = os.path.dirname(ARCHIVE)\nt0 = time.time()\n\n# --- 1. Get every file name in the archive along with its size (in archive order)\nlisting = subprocess.run([\"7z\", \"l\", \"-slt\", ARCHIVE],\n                         capture_output=True, text=True, check=True).stdout\nbody = listing.split(\"\\n----------\\n\", 1)[1]\nentries = []\nfor block in body.split(\"\\n\\n\"):\n    info = dict(line.split(\" = \", 1)\n                for line in block.strip().splitlines() if \" = \" in line)\n    if info.get(\"Path\", \"\").endswith(\".png\"):\n        entries.append((info[\"Path\"], int(info[\"Size\"])))\nprint(\"Files in archive:\", len(entries), flush=True)\nif len(entries) != 300000:\n    raise RuntimeError(f\"Archive listing found {len(entries)} files, expected 300000.\")\n\n# --- 2. Stream the unpacked bytes and decode each PNG without touching disk\nimages = np.zeros((300000, 32, 32, 3), dtype=np.uint8)\nproc = subprocess.Popen([\"7z\", \"x\", \"-so\", ARCHIVE],\n                        stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,\n                        bufsize=1 << 20)\nfor k, (path, size) in enumerate(entries):\n    data = proc.stdout.read(size)\n    idx = int(os.path.basename(path)[:-4]) - 1          # file \"123.png\" goes to row 122\n    images[idx] = np.asarray(Image.open(io.BytesIO(data)).convert(\"RGB\"))\n    if (k + 1) % 50000 == 0:\n        print(f\"  {k + 1:,} images decoded ({time.time() - t0:.0f}s)\", flush=True)\nproc.wait()\nprint(f\"All images loaded in {time.time() - t0:.0f}s\", flush=True)\n\n# --- 3. Predict on each image and its mirror image, then average, working in chunks\ndef predict_tta(X, chunk=20000):\n    out = []\n    for i in range(0, len(X), chunk):\n        xb = X[i:i + chunk].astype(\"float32\")\n        p = model.predict(xb, batch_size=1000, verbose=0)\n        p += model.predict(np.ascontiguousarray(xb[:, :, ::-1]),\n                           batch_size=1000, verbose=0)\n        out.append(p / 2)\n        print(f\"  predicted {min(i + chunk, len(X)):,} / {len(X):,}\", flush=True)\n    return np.concatenate(out)\n\nprobs = predict_tta(images)\n\n# --- 4. Save submission.csv\nsub = pd.read_csv(os.path.join(COMP_DIR, \"sampleSubmission.csv\"))\nsub[\"label\"] = [CLASSES[i] for i in probs[sub[\"id\"].values - 1].argmax(axis=1)]\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)\nprint(sub.head())\nprint(\"\\nPredicted class counts (should be roughly balanced):\")\nprint(sub[\"label\"].value_counts().to_string())\nprint(f\"\\nsubmission.csv written: {len(sub):,} rows, total {time.time() - t0:.0f}s\")\n\n# --- 5. Training curves and a summary line for the report\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4))\nax1.plot(history.history[\"accuracy\"], label=\"train\")\nax1.plot(history.history[\"val_accuracy\"], label=\"test\")\nax1.set_xlabel(\"epoch\"); ax1.set_ylabel(\"accuracy\"); ax1.set_title(\"Accuracy\"); ax1.legend()\nax2.plot(history.history[\"loss\"], label=\"train\")\nax2.plot(history.history[\"val_loss\"], label=\"test\")\nax2.set_xlabel(\"epoch\"); ax2.set_ylabel(\"loss\"); ax2.set_title(\"Loss\"); ax2.legend()\nplt.tight_layout(); plt.show()\n\nprint(f\"Summary: ResNet-9, {model.count_params():,} parameters, {EPOCHS} epochs, \"\n      f\"{train_minutes:.1f} min training, official test accuracy {test_acc * 100:.2f}%\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}