{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CAPTCHA Security Evaluation Against Automated Recognition Attacks\n### Deep Learning-Based Evaluation using a CNN + BiLSTM + CTC Model\n\n**Authors:** Bander Al-Eidan, Almgdad Mohammed\n\n**Course:** CSC621 – Research Methods\n\n**Instructor:** Dr. Haifa\n\n**Institution:** Qassim University – 2026\n\n---\n\n**Datasets:**\n- DS1: CAPTCHA Images (Kaggle, fournierp)\n- DS2: Break the CAPTCHA Challenge (Kaggle competition)\n- DS3: Synthetic CAPTCHAs generated in this notebook with the Python `captcha` library","metadata":{}},{"cell_type":"code","source":"# ==== Cell 1: Environment check ====\nimport sys, platform\nimport tensorflow as tf\nprint(\"Python version     :\", sys.version.split()[0])\nprint(\"TensorFlow version :\", tf.__version__)\nprint(\"Keras version      :\", tf.keras.__version__ if hasattr(tf.keras, \"__version__\") else \"bundled with TF\")\nprint(\"Platform           :\", platform.platform())\ngpus = tf.config.list_physical_devices(\"GPU\")\nprint(\"GPUs found         :\", gpus)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:45:09.693162Z","iopub.execute_input":"2026-10-12T17:45:09.693841Z","iopub.status.idle":"2026-10-12T17:45:27.995905Z","shell.execute_reply.started":"2026-10-12T17:45:09.693809Z","shell.execute_reply":"2026-10-12T17:45:27.995068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 2: Install the captcha library (used to generate DS3) ====\n!pip install -q captcha\nimport captcha\nprint(\"captcha version:\", getattr(captcha, \"__version__\", \"unknown\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:45:27.997473Z","iopub.execute_input":"2026-10-12T17:45:27.99834Z","iopub.status.idle":"2026-10-12T17:45:32.07052Z","shell.execute_reply.started":"2026-10-12T17:45:27.998316Z","shell.execute_reply":"2026-10-12T17:45:32.069601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 3: Locate the three fonts for DS3 ====\nimport glob, os\n!apt-get -qq install -y fonts-dejavu-core fonts-liberation fonts-freefont-ttf > /dev/null 2>&1\nwanted = {\"DejaVu Sans\": \"DejaVuSans.ttf\", \"Liberation Serif\": \"LiberationSerif-Regular.ttf\", \"FreeMono\": \"FreeMono.ttf\"}\nFONT_PATHS = {}\nfor name, filename in wanted.items():\n    found = glob.glob(f\"/usr/share/fonts/**/{filename}\", recursive=True)\n    FONT_PATHS[name] = found[0] if found else None\n    print(f\"{name:17s} -> {FONT_PATHS[name]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:45:32.072027Z","iopub.execute_input":"2026-10-12T17:45:32.072922Z","iopub.status.idle":"2026-10-12T17:45:39.99208Z","shell.execute_reply.started":"2026-10-12T17:45:32.07289Z","shell.execute_reply":"2026-10-12T17:45:39.991322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 4: Fix the random seed ====\nimport random\nimport numpy as np\nSEED = 50\nrandom.seed(SEED); np.random.seed(SEED); tf.random.set_seed(SEED)\nos.environ[\"PYTHONHASHSEED\"] = str(SEED)\nprint(\"Seed fixed to\", SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:45:39.993316Z","iopub.execute_input":"2026-10-12T17:45:39.993593Z","iopub.status.idle":"2026-10-12T17:45:40.000111Z","shell.execute_reply.started":"2026-10-12T17:45:39.99357Z","shell.execute_reply":"2026-10-12T17:45:39.999346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 5: Inspect the two Kaggle inputs ====\nfrom collections import Counter\nfor root in sorted(glob.glob(\"/kaggle/input/*\")):\n    print(\"\\n==\", root)\n    print(\"   Top-level items:\")\n    for item in sorted(os.listdir(root))[:10]:\n        path = os.path.join(root, item)\n        kind = \"folder\" if os.path.isdir(path) else \"file\"\n        print(\"     \", kind, \"-\", item)\n    ext_count = Counter()\n    for dirpath, _, files in os.walk(root):\n        for f in files:\n            ext_count[os.path.splitext(f)[1].lower()] += 1\n    print(\"   Files by type:\", dict(ext_count))\n    print(\"   Total files  :\", sum(ext_count.values()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:45:40.002144Z","iopub.execute_input":"2026-10-12T17:45:40.002522Z","iopub.status.idle":"2026-10-12T17:46:17.629559Z","shell.execute_reply.started":"2026-10-12T17:45:40.0025Z","shell.execute_reply":"2026-10-12T17:46:17.628945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 6: Look inside the folders ====\nfor root in sorted(glob.glob(\"/kaggle/input/*\")):\n    print(\"\\n==\", root)\n    for dirpath, dirnames, files in os.walk(root):\n        if files:\n            print(f\"   {len(files):6d} files  in  {dirpath}\")\n\nprint(\"\\n===== CSV / TXT files in DS2 =====\")\nfor f in glob.glob(\"/kaggle/input/competitions/**/*.csv\", recursive=True) + \\\n         glob.glob(\"/kaggle/input/competitions/**/*.txt\", recursive=True):\n    print(\"\\n--\", f)\n    with open(f) as fh:\n        for i, line in enumerate(fh):\n            if i >= 5: break\n            print(\"   \", line.rstrip())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:46:17.630614Z","iopub.execute_input":"2026-10-12T17:46:17.630987Z","iopub.status.idle":"2026-10-12T17:46:53.430484Z","shell.execute_reply.started":"2026-10-12T17:46:17.630966Z","shell.execute_reply":"2026-10-12T17:46:53.429796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 7: Build (image path, label) lists for DS1 and DS2 ====\nimport pandas as pd\nDS1_DIR = \"/kaggle/input/datasets/fournierp/captcha-version-2-images/samples\"\nDS2_DIR = \"/kaggle/input/competitions/break-the-captcha-challenge/dataset\"\n\n# DS1: label = file name without extension (files directly in DS1_DIR only -> skips the duplicated sub-folder)\nds1 = []\nfor f in sorted(os.listdir(DS1_DIR)):\n    p = os.path.join(DS1_DIR, f)\n    if os.path.isfile(p):\n        ds1.append((p, os.path.splitext(f)[0]))\n\n# DS2: labels come from train_metadata.csv (read as text to keep leading zeros)\nmeta = pd.read_csv(os.path.join(DS2_DIR, \"train_metadata.csv\"), dtype=str)\nds2 = [(os.path.join(DS2_DIR, \"train\", n), t) for t, n in zip(meta[\"captcha_text\"], meta[\"image_name\"])]\n\ndef describe(name, data):\n    labels  = [l for _, l in data]\n    lengths = Counter(len(l) for l in labels)\n    chars   = sorted(set(\"\".join(labels)))\n    missing = sum(not os.path.exists(p) for p, _ in data)\n    print(f\"\\n{name}: {len(data)} images | missing files: {missing}\")\n    print(\"   label lengths :\", dict(lengths))\n    print(f\"   characters ({len(chars)}):\", \"\".join(chars))\n    print(\"   examples      :\", labels[:5])\n\ndescribe(\"DS1\", ds1)\ndescribe(\"DS2\", ds2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:46:53.431525Z","iopub.execute_input":"2026-10-12T17:46:53.431817Z","iopub.status.idle":"2026-10-12T17:46:56.295858Z","shell.execute_reply.started":"2026-10-12T17:46:53.431787Z","shell.execute_reply":"2026-10-12T17:46:56.295219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 8: DS3 (synthetic) - generated ONCE (Version 1), then loaded from the saved output ====\nfrom captcha.image import ImageCaptcha\nimport string\n\nGENERATE_DS3 = False   # True was used ONCE, in Version 1. The captcha library uses the `secrets` module,\n                       # so images cannot be regenerated identically -> we always reuse the saved Version-1 images.\nCHARSET = string.ascii_uppercase + string.digits   # A-Z and 0-9 (36 symbols), same as DS2\nN_DS3, LEN = 10000, 5\n\nif GENERATE_DS3:\n    DS3_DIR = \"/kaggle/working/ds3\"\n    os.makedirs(DS3_DIR, exist_ok=True)\n    # The ONLY change from the library defaults: three fonts instead of DroidSansMono.\n    generator = ImageCaptcha(fonts=list(FONT_PATHS.values()))\n    rng = random.Random(SEED)   # makes the label texts reproducible (seed 50)\n    rows = []\n    for i in range(N_DS3):\n        text = \"\".join(rng.choice(CHARSET) for _ in range(LEN))\n        generator.write(text, os.path.join(DS3_DIR, f\"{i}.png\"))\n        rows.append((text, f\"{i}.png\"))\n    pd.DataFrame(rows, columns=[\"captcha_text\", \"image_name\"]).to_csv(\n        os.path.join(DS3_DIR, \"ds3_metadata.csv\"), index=False)\nelse:\n    csv_path = glob.glob(\"/kaggle/input/**/ds3/ds3_metadata.csv\", recursive=True)[0]\n    DS3_DIR = os.path.dirname(csv_path)\n    print(\"Loading saved DS3 from:\", DS3_DIR)\n\nmeta3 = pd.read_csv(os.path.join(DS3_DIR, \"ds3_metadata.csv\"), dtype=str)\nds3 = [(os.path.join(DS3_DIR, n), t) for t, n in zip(meta3[\"captcha_text\"], meta3[\"image_name\"])]\ndescribe(\"DS3\", ds3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:46:56.296696Z","iopub.execute_input":"2026-10-12T17:46:56.297016Z","iopub.status.idle":"2026-10-12T17:47:01.349009Z","shell.execute_reply.started":"2026-10-12T17:46:56.296995Z","shell.execute_reply":"2026-10-12T17:47:01.348301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 9: Show 4 sample images from each dataset ====\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nfig, axes = plt.subplots(3, 4, figsize=(14, 5))\nfor row, (name, data) in enumerate([(\"DS1\", ds1), (\"DS2\", ds2), (\"DS3\", ds3)]):\n    for col, (path, label) in enumerate(random.Random(SEED).sample(data, 4)):\n        ax = axes[row, col]\n        ax.imshow(Image.open(path))\n        ax.set_title(f\"{name}: {label}\")\n        ax.axis(\"off\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:47:01.350004Z","iopub.execute_input":"2026-10-12T17:47:01.350302Z","iopub.status.idle":"2026-10-12T17:47:02.031475Z","shell.execute_reply.started":"2026-10-12T17:47:01.35028Z","shell.execute_reply":"2026-10-12T17:47:02.030651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 10: Sample 1,000 images per dataset, uppercase labels, split 70/15/15 ====\nN_SAMPLE = 1000\nsplits = {}\nfor name, data in [(\"DS1\", ds1), (\"DS2\", ds2), (\"DS3\", ds3)]:\n    sample = random.Random(SEED).sample(data, N_SAMPLE)   # random but reproducible (seed 50)\n    sample = [(p, l.upper()) for p, l in sample]          # label normalization: lowercase -> uppercase\n    splits[name] = {\"train\": sample[:700], \"val\": sample[700:850], \"test\": sample[850:]}\n\n    # safety check: no image may appear in two subsets (prevents data leakage)\n    tr, va, te = (set(p for p, _ in splits[name][k]) for k in (\"train\", \"val\", \"test\"))\n    assert not (tr & va or tr & te or va & te), f\"{name}: overlap found!\"\n\n    print(f\"{name}: train={len(tr)}, val={len(va)}, test={len(te)} | no overlap ✓ | \"\n          f\"examples: {[l for _, l in sample[:3]]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:47:02.03247Z","iopub.execute_input":"2026-10-12T17:47:02.032813Z","iopub.status.idle":"2026-10-12T17:47:02.042931Z","shell.execute_reply.started":"2026-10-12T17:47:02.032782Z","shell.execute_reply":"2026-10-12T17:47:02.042054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 11: Image preprocessing (resize, grayscale, padding, 0-1) ====\nIMG_W, IMG_H = 160, 60\n\ndef preprocess(path):\n    img = Image.open(path).convert(\"RGB\")\n    if img.size != (IMG_W, IMG_H):                    # only DS1 images (200x50)\n        img = img.resize((160, 40), Image.BILINEAR)   # 1) scale down, keeping the 4:1 aspect ratio\n    img = img.convert(\"L\")                            # 2) grayscale, 1 channel\n    arr = np.asarray(img, dtype=np.float32)\n    if arr.shape[0] != IMG_H:                         # 3) pad 10 px top and bottom (DS1 only)\n        arr = np.pad(arr, ((10, 10), (0, 0)), mode=\"edge\")   #    using the background colour at the edge\n    return arr / 255.0                                # 4) pixel normalization to 0-1\n\ndata = {}\nfor name in splits:\n    data[name] = {}\n    for part, items in splits[name].items():\n        X = np.stack([preprocess(p) for p, _ in items])[..., np.newaxis]   # shape: (n, 60, 160, 1)\n        y = [l for _, l in items]\n        data[name][part] = (X, y)\n    Xtr = data[name][\"train\"][0]\n    print(f\"{name}: X_train shape {Xtr.shape} | min={Xtr.min():.2f}, max={Xtr.max():.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:47:02.043984Z","iopub.execute_input":"2026-10-12T17:47:02.04462Z","iopub.status.idle":"2026-10-12T17:47:11.705831Z","shell.execute_reply.started":"2026-10-12T17:47:02.044599Z","shell.execute_reply":"2026-10-12T17:47:11.704942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 12: Show preprocessed images (what the model will actually see) ====\nfig, axes = plt.subplots(3, 4, figsize=(14, 5))\nfor row, name in enumerate([\"DS1\", \"DS2\", \"DS3\"]):\n    X, y = data[name][\"train\"]\n    for col in range(4):\n        ax = axes[row, col]\n        ax.imshow(X[col, :, :, 0], cmap=\"gray\", vmin=0, vmax=1)\n        ax.set_title(f\"{name}: {y[col]}  ({X.shape[2]}x{X.shape[1]})\")\n        ax.axis(\"off\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:47:11.706821Z","iopub.execute_input":"2026-10-12T17:47:11.70723Z","iopub.status.idle":"2026-10-12T17:47:12.294872Z","shell.execute_reply.started":"2026-10-12T17:47:11.707205Z","shell.execute_reply":"2026-10-12T17:47:12.294081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 13: Encode labels as numbers (the model works with numbers, not letters) ====\nCHARS = string.digits + string.ascii_uppercase      # 36 characters: 0-9 then A-Z\nchar_to_num = {c: i for i, c in enumerate(CHARS)}   # '0'->0 ... '9'->9, 'A'->10 ... 'Z'->35\nnum_to_char = {i: c for c, i in char_to_num.items()}\nBLANK = len(CHARS)                                  # 36 = the extra CTC \"blank\" symbol\nNUM_CLASSES = len(CHARS) + 1                        # 37 output units (26 letters + 10 digits + blank)\n\ndef encode(labels):\n    return np.array([[char_to_num[c] for c in l] for l in labels], dtype=np.int32)\n\nfor name in data:\n    for part in (\"train\", \"val\", \"test\"):\n        X, y = data[name][part][:2]\n        data[name][part] = (X, y, encode(y))\n\nprint(\"Example :\", data[\"DS2\"][\"train\"][1][0], \"->\", data[\"DS2\"][\"train\"][2][0])\nprint(\"Labels shape (DS1 train):\", data[\"DS1\"][\"train\"][2].shape)\nprint(\"Output classes:\", NUM_CLASSES)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:51:34.091401Z","iopub.execute_input":"2026-10-12T17:51:34.09193Z","iopub.status.idle":"2026-10-12T17:51:34.103856Z","shell.execute_reply.started":"2026-10-12T17:51:34.091902Z","shell.execute_reply":"2026-10-12T17:51:34.103085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 14: Build the CNN + BiLSTM model (Section 5.5 / Table 4) ====\nimport keras\nfrom keras import layers\n\ndef build_model():\n    inp = keras.Input(shape=(IMG_H, IMG_W, 1), name=\"image\")                  # (60, 160, 1)\n    x = inp\n    for filters in (32, 64, 128):                                              # 3 convolutional blocks\n        x = layers.Conv2D(filters, (3, 3), strides=(1, 1), padding=\"same\", activation=\"relu\")(x)\n        x = layers.MaxPooling2D((2, 2))(x)\n    # x is now (7, 20, 128): make the WIDTH the time axis -> 20 time steps of 7*128 = 896 features\n    x = layers.Permute((2, 1, 3))(x)                                           # (20, 7, 128)\n    x = layers.Reshape((x.shape[1], x.shape[2] * x.shape[3]))(x)              # (20, 896)\n    x = layers.Bidirectional(layers.LSTM(128, return_sequences=True))(x)      # (20, 256)\n    out = layers.Dense(NUM_CLASSES, activation=\"softmax\", name=\"chars\")(x)    # (20, 37)\n    return keras.Model(inp, out, name=\"CNN_BiLSTM_CTC\")\n\nkeras.utils.set_random_seed(SEED)   # same initial weights every time\nmodel = build_model()\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:54:12.801461Z","iopub.execute_input":"2026-10-12T17:54:12.801991Z","iopub.status.idle":"2026-10-12T17:54:14.942384Z","shell.execute_reply.started":"2026-10-12T17:54:12.801958Z","shell.execute_reply":"2026-10-12T17:54:14.941722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 15: CTC loss + greedy CTC decoding + quick test (model NOT trained yet) ====\nimport tensorflow as tf\n\ndef ctc_loss(y_true, y_pred):\n    y_true = tf.cast(y_true, tf.int32)                     # (batch, 5) true characters as numbers\n    batch = tf.shape(y_pred)[0]\n    label_len = tf.fill([batch], tf.shape(y_true)[1])      # every label has 5 characters\n    logit_len = tf.fill([batch], tf.shape(y_pred)[1])      # every prediction has 20 time steps\n    loss = tf.nn.ctc_loss(labels=y_true, logits=tf.math.log(y_pred + 1e-7),\n                          label_length=label_len, logit_length=logit_len,\n                          logits_time_major=False, blank_index=BLANK)\n    return tf.reduce_mean(loss)\n\ndef greedy_decode(probs):\n    texts = []\n    for seq in np.argmax(probs, axis=-1):        # 1) most likely symbol at each of the 20 time steps\n        out, prev = [], None\n        for k in seq:\n            if k != prev and k != BLANK:        # 2) merge repeated symbols  3) remove blanks\n                out.append(num_to_char[int(k)])\n            prev = k\n        texts.append(\"\".join(out))\n    return texts\n\n# quick test on 8 DS2 training images\nXb, yb, lb = (a[:8] for a in data[\"DS2\"][\"train\"])\npb = model.predict(Xb, verbose=0)\nprint(\"Output shape :\", pb.shape)\nprint(\"CTC loss     :\", round(float(ctc_loss(lb, pb)), 2))\nprint(\"True labels  :\", list(yb))\nprint(\"Predictions  :\", greedy_decode(pb))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T17:59:14.988555Z","iopub.execute_input":"2026-10-12T17:59:14.989059Z","iopub.status.idle":"2026-10-12T17:59:17.8219Z","shell.execute_reply.started":"2026-10-12T17:59:14.989029Z","shell.execute_reply":"2026-10-12T17:59:17.820967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 16: Training function (Section 5.7) + train on DS1 ====\nimport time\nos.makedirs(\"/kaggle/working/models\", exist_ok=True)\nhistories, models, train_info = {}, {}, {}\n\ndef train_on(name):\n    keras.utils.set_random_seed(SEED)       # same initial weights for every dataset (fair comparison)\n    m = build_model()                        # a NEW model for each dataset\n    m.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=ctc_loss)\n    early = keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, restore_best_weights=True)\n    Xtr, _, Ltr = data[name][\"train\"]\n    Xva, _, Lva = data[name][\"val\"]          # the TEST set is NOT used here\n    t0 = time.time()\n    hist = m.fit(Xtr, Ltr, validation_data=(Xva, Lva), batch_size=32, epochs=50,\n                 callbacks=[early], verbose=2)\n    secs = time.time() - t0\n    m.save_weights(f\"/kaggle/working/models/{name}.weights.h5\")\n    h = hist.history\n    best = int(np.argmin(h[\"val_loss\"])) + 1\n    train_info[name] = {\"epochs_run\": len(h[\"loss\"]), \"best_epoch\": best,\n                        \"best_val_loss\": float(min(h[\"val_loss\"])), \"seconds\": round(secs)}\n    print(f\"\\n{name}: ran {len(h['loss'])} epochs | best epoch = {best} | \"\n          f\"best val_loss = {min(h['val_loss']):.3f} | time = {secs:.0f} s\")\n    return m, h\n\nmodels[\"DS1\"], histories[\"DS1\"] = train_on(\"DS1\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T18:03:54.061583Z","iopub.execute_input":"2026-10-12T18:03:54.062232Z","iopub.status.idle":"2026-10-12T18:04:58.555968Z","shell.execute_reply.started":"2026-10-12T18:03:54.062203Z","shell.execute_reply":"2026-10-12T18:04:58.555243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 17: Train on DS2 and DS3 (same function, same settings) ====\nfor name in (\"DS2\", \"DS3\"):\n    print(f\"\\n==================== Training on {name} ====================\")\n    models[name], histories[name] = train_on(name)\n\nprint(\"\\n===== Training summary =====\")\nfor name, info in train_info.items():\n    print(name, info)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T18:11:52.183698Z","iopub.execute_input":"2026-10-12T18:11:52.184191Z","iopub.status.idle":"2026-10-12T18:13:10.56858Z","shell.execute_reply.started":"2026-10-12T18:11:52.184163Z","shell.execute_reply":"2026-10-12T18:13:10.567591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 18: Experiment 1 - evaluation on the TEST set (used only once) ====\ndef edit_distance(a, b):\n    \"\"\"Minimum number of substitutions, deletions and insertions (S + D + I).\"\"\"\n    d = list(range(len(b) + 1))\n    for i in range(1, len(a) + 1):\n        prev, d[0] = d[0], i\n        for j in range(1, len(b) + 1):\n            cur = d[j]\n            d[j] = min(d[j] + 1, d[j - 1] + 1, prev + (a[i - 1] != b[j - 1]))\n            prev = cur\n    return d[len(b)]\n\ndef evaluate(m, name):\n    X, y, _ = data[name][\"test\"]\n    preds = greedy_decode(m.predict(X, verbose=0))\n    errors = sum(edit_distance(p, t) for p, t in zip(preds, y))   # S + D + I\n    n_chars = sum(len(t) for t in y)                               # N\n    char_acc = 1 - errors / n_chars\n    captcha_acc = float(np.mean([p == t for p, t in zip(preds, y)]))\n    return char_acc, captcha_acc, preds\n\nresults_exp1 = {}\nfor name in (\"DS1\", \"DS2\", \"DS3\"):\n    ca, wa, preds = evaluate(models[name], name)\n    n_ok = sum(p == t for p, t in zip(preds, data[name][\"test\"][1]))\n    results_exp1[name] = {\"char_acc\": ca, \"captcha_acc\": wa, \"correct\": n_ok, \"preds\": preds}\n    status = \"BROKEN\" if wa >= 0.01 else \"NOT broken\"\n    print(f\"{name}: Character Acc = {ca*100:6.2f}% | CAPTCHA-level Acc = {wa*100:6.2f}% \"\n          f\"({n_ok}/150) -> {status}\")\n\nprint(\"\\nExamples (true -> predicted):\")\nfor name in (\"DS1\", \"DS2\", \"DS3\"):\n    pairs = list(zip(data[name][\"test\"][1][:5], results_exp1[name][\"preds\"][:5]))\n    print(f\"  {name}: {pairs}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T18:29:59.563167Z","iopub.execute_input":"2026-10-12T18:29:59.563969Z","iopub.status.idle":"2026-10-12T18:30:00.989967Z","shell.execute_reply.started":"2026-10-12T18:29:59.563936Z","shell.execute_reply":"2026-10-12T18:30:00.989188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 19: Experiment 2 - larger equal samples for DS2 and DS3 (10,000 each) ====\nN_SAMPLE_2 = 10000\ndata2 = {}\nfor name, src in [(\"DS2\", ds2), (\"DS3\", ds3)]:\n    sample = random.Random(SEED).sample(src, N_SAMPLE_2)          # same procedure, seed 50\n    sample = [(p, l.upper()) for p, l in sample]\n    parts = {\"train\": sample[:7000], \"val\": sample[7000:8500], \"test\": sample[8500:]}   # 70/15/15\n\n    tr, va, te = (set(p for p, _ in parts[k]) for k in (\"train\", \"val\", \"test\"))\n    assert not (tr & va or tr & te or va & te), f\"{name}: overlap found!\"\n\n    data2[name] = {}\n    for part, items in parts.items():\n        X = np.stack([preprocess(p) for p, _ in items])[..., np.newaxis]   # same preprocessing\n        y = [l for _, l in items]\n        data2[name][part] = (X, y, encode(y))\n    print(f\"{name}: train={len(tr)}, val={len(va)}, test={len(te)} | no overlap ✓ | \"\n          f\"X_train shape {data2[name]['train'][0].shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T18:34:50.208154Z","iopub.execute_input":"2026-10-12T18:34:50.208922Z","iopub.status.idle":"2026-10-12T18:36:20.759428Z","shell.execute_reply.started":"2026-10-12T18:34:50.208891Z","shell.execute_reply":"2026-10-12T18:36:20.75879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 20: Experiment 2 - train on DS2 and DS3 (7,000 training images each), SAME settings ====\nmodels2, histories2, train_info2 = {}, {}, {}\n\ndef train_model(name, dataset, info, tag):\n    keras.utils.set_random_seed(SEED)       # same initial weights as in Experiment 1\n    m = build_model()\n    m.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=ctc_loss)\n    early = keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, restore_best_weights=True)\n    Xtr, _, Ltr = dataset[name][\"train\"]\n    Xva, _, Lva = dataset[name][\"val\"]       # the TEST set is NOT used here\n    t0 = time.time()\n    hist = m.fit(Xtr, Ltr, validation_data=(Xva, Lva), batch_size=32, epochs=50,\n                 callbacks=[early], verbose=2)\n    secs = time.time() - t0\n    m.save_weights(f\"/kaggle/working/models/{name}_{tag}.weights.h5\")\n    h = hist.history\n    best = int(np.argmin(h[\"val_loss\"])) + 1\n    info[name] = {\"epochs_run\": len(h[\"loss\"]), \"best_epoch\": best,\n                  \"best_val_loss\": float(min(h[\"val_loss\"])), \"seconds\": round(secs)}\n    print(f\"\\n{name} ({tag}): ran {len(h['loss'])} epochs | best epoch = {best} | \"\n          f\"best val_loss = {min(h['val_loss']):.3f} | time = {secs:.0f} s\")\n    return m, h\n\nfor name in (\"DS2\", \"DS3\"):\n    print(f\"\\n==================== Experiment 2: training on {name} ====================\")\n    models2[name], histories2[name] = train_model(name, data2, train_info2, \"exp2\")\n\nprint(\"\\n===== Experiment 2 training summary =====\")\nfor name, info in train_info2.items():\n    print(name, info)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T18:38:58.345368Z","iopub.execute_input":"2026-10-12T18:38:58.34611Z","iopub.status.idle":"2026-10-12T18:49:50.251099Z","shell.execute_reply.started":"2026-10-12T18:38:58.346079Z","shell.execute_reply":"2026-10-12T18:49:50.250336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 21: Experiment 2 - evaluation on the TEST set (1,500 images each, used only once) ====\ndef evaluate_on(m, dataset, name):\n    X, y, _ = dataset[name][\"test\"]\n    preds = greedy_decode(m.predict(X, verbose=0))\n    errors = sum(edit_distance(p, t) for p, t in zip(preds, y))   # S + D + I\n    n_chars = sum(len(t) for t in y)                               # N\n    char_acc = 1 - errors / n_chars\n    n_ok = sum(p == t for p, t in zip(preds, y))\n    return char_acc, n_ok / len(y), n_ok, preds\n\nresults_exp2 = {}\nfor name in (\"DS2\", \"DS3\"):\n    ca, wa, n_ok, preds = evaluate_on(models2[name], data2, name)\n    results_exp2[name] = {\"char_acc\": ca, \"captcha_acc\": wa, \"correct\": n_ok, \"preds\": preds}\n    status = \"BROKEN\" if wa >= 0.01 else \"NOT broken\"\n    print(f\"{name}: Character Acc = {ca*100:6.2f}% | CAPTCHA-level Acc = {wa*100:6.2f}% \"\n          f\"({n_ok}/1500) -> {status}\")\n\nprint(\"\\nExamples of WRONG predictions (true -> predicted):\")\nfor name in (\"DS2\", \"DS3\"):\n    y = data2[name][\"test\"][1]\n    wrong = [(t, p) for t, p in zip(y, results_exp2[name][\"preds\"]) if t != p][:6]\n    print(f\"  {name}: {wrong}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T18:53:51.268703Z","iopub.execute_input":"2026-10-12T18:53:51.269065Z","iopub.status.idle":"2026-10-12T18:53:52.990468Z","shell.execute_reply.started":"2026-10-12T18:53:51.269041Z","shell.execute_reply":"2026-10-12T18:53:52.989888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 22: Experiment 2 - is the DS2 vs DS3 difference statistically significant? ====\n# Two-proportion z-test on CAPTCHA-level Accuracy (1,500 test images each)\nfrom math import sqrt, erf\nn1 = n2 = 1500\nx1, x2 = results_exp2[\"DS2\"][\"correct\"], results_exp2[\"DS3\"][\"correct\"]\np1, p2 = x1 / n1, x2 / n2\np_pool = (x1 + x2) / (n1 + n2)\nse = sqrt(p_pool * (1 - p_pool) * (1 / n1 + 1 / n2))\nz = (p2 - p1) / se\np_value = 2 * (1 - 0.5 * (1 + erf(abs(z) / sqrt(2))))\n\nci1 = 1.96 * sqrt(p1 * (1 - p1) / n1)   # 95% confidence interval (half-width)\nci2 = 1.96 * sqrt(p2 * (1 - p2) / n2)\nprint(f\"DS2 = {p1*100:.2f}% ± {ci1*100:.2f} | DS3 = {p2*100:.2f}% ± {ci2*100:.2f} (95% CI)\")\nprint(f\"Difference (DS3 - DS2) = {(p2 - p1)*100:+.2f} percentage points\")\nprint(f\"z = {z:.2f} | p-value = {p_value:.3f} ->\",\n      \"SIGNIFICANT (p < 0.05)\" if p_value < 0.05 else \"NOT significant (p >= 0.05)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T19:02:16.728192Z","iopub.execute_input":"2026-10-12T19:02:16.728637Z","iopub.status.idle":"2026-10-12T19:02:16.735728Z","shell.execute_reply.started":"2026-10-12T19:02:16.728605Z","shell.execute_reply":"2026-10-12T19:02:16.735139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 23: Save ALL results + training histories (nothing is lost when the session stops) ====\nimport json\n\ndef as_floats(h):\n    return {k: [float(v) for v in vals] for k, vals in h.items()}\n\ndef pack(name, info, hist, res, dataset):\n    return {\"train_info\": info[name], \"history\": as_floats(hist[name]),\n            \"char_acc\": float(res[name][\"char_acc\"]), \"captcha_acc\": float(res[name][\"captcha_acc\"]),\n            \"correct\": int(res[name][\"correct\"]),\n            \"test_labels\": list(dataset[name][\"test\"][1]), \"test_preds\": list(res[name][\"preds\"])}\n\nall_results = {\n    \"exp1\": {n: pack(n, train_info, histories, results_exp1, data) for n in (\"DS1\", \"DS2\", \"DS3\")},\n    \"exp2\": {n: pack(n, train_info2, histories2, results_exp2, data2) for n in (\"DS2\", \"DS3\")},\n    \"exp2_significance\": {\"z\": float(z), \"p_value\": float(p_value)},\n}\nwith open(\"/kaggle/working/results_all.json\", \"w\") as f:\n    json.dump(all_results, f)\n\nprint(\"Saved to /kaggle/working/results_all.json\")\nprint(json.dumps(all_results))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-12T19:07:50.288813Z","iopub.execute_input":"2026-10-12T19:07:50.289264Z","iopub.status.idle":"2026-10-12T19:07:50.30399Z","shell.execute_reply.started":"2026-10-12T19:07:50.289235Z","shell.execute_reply":"2026-10-12T19:07:50.303053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 24: Load the saved results (from Version 2) and build the results tables ====\nimport json, glob\nimport pandas as pd\n\nres_path = glob.glob(\"/kaggle/input/**/results_all.json\", recursive=True)[0]\nwith open(res_path) as f:\n    R = json.load(f)\nprint(\"Loaded:\", res_path, \"\\n\")\n\ndef table(exp, names, n_test):\n    rows = []\n    for n in names:\n        r = R[exp][n]\n        rows.append({\n            \"Dataset\": n,\n            \"Epochs run\": r[\"train_info\"][\"epochs_run\"],\n            \"Best epoch\": r[\"train_info\"][\"best_epoch\"],\n            \"Best val loss\": round(r[\"train_info\"][\"best_val_loss\"], 3),\n            \"Training time (s)\": r[\"train_info\"][\"seconds\"],\n            \"Character Accuracy (%)\": round(r[\"char_acc\"] * 100, 2),\n            \"CAPTCHA-level Accuracy (%)\": round(r[\"captcha_acc\"] * 100, 2),\n            \"Correct / Tested\": f'{r[\"correct\"]}/{n_test}',\n            \"Broken (>= 1%)?\": \"Yes\" if r[\"captcha_acc\"] >= 0.01 else \"No\",\n        })\n    return pd.DataFrame(rows)\n\nt1 = table(\"exp1\", [\"DS1\", \"DS2\", \"DS3\"], 150)\nt2 = table(\"exp2\", [\"DS2\", \"DS3\"], 1500)\nt1.to_csv(\"/kaggle/working/table_experiment1.csv\", index=False)\nt2.to_csv(\"/kaggle/working/table_experiment2.csv\", index=False)\n\nprint(\"Experiment 1 (1,000 images per dataset: 700 train / 150 val / 150 test)\")\ndisplay(t1)\nprint(\"\\nExperiment 2 (10,000 images per dataset: 7,000 train / 1,500 val / 1,500 test)\")\ndisplay(t2)\ns = R[\"exp2_significance\"]\nprint(f\"\\nDS2 vs DS3 (Experiment 2): z = {s['z']:.2f}, p = {s['p_value']:.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-13T14:09:28.090499Z","iopub.execute_input":"2026-10-13T14:09:28.090813Z","iopub.status.idle":"2026-10-13T14:10:15.001591Z","shell.execute_reply.started":"2026-10-13T14:09:28.090786Z","shell.execute_reply":"2026-10-13T14:10:15.000703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 25: Learning curves of all 5 training runs (figure for the paper) ====\nimport matplotlib.pyplot as plt\n\nCOLORS = {\"DS1\": \"#2a78d6\", \"DS2\": \"#eb6834\", \"DS3\": \"#1baf7a\"}   # one fixed colour per dataset\nplt.rcParams.update({\"font.size\": 11, \"axes.spines.top\": False, \"axes.spines.right\": False})\nfig, axes = plt.subplots(1, 2, figsize=(13, 4.6), sharey=True)\npanels = [(\"exp1\", [\"DS1\", \"DS2\", \"DS3\"], \"Experiment 1 (700 training images per dataset)\"),\n          (\"exp2\", [\"DS2\", \"DS3\"], \"Experiment 2 (7,000 training images per dataset)\")]\n\nfor ax, (exp, names, title) in zip(axes, panels):\n    for n in names:\n        h = R[exp][n][\"history\"]\n        ep = range(1, len(h[\"loss\"]) + 1)\n        ax.plot(ep, h[\"val_loss\"], color=COLORS[n], lw=2, label=f\"{n} validation\")\n        ax.plot(ep, h[\"loss\"], color=COLORS[n], lw=1.5, ls=\"--\", alpha=0.8, label=f\"{n} training\")\n        b = R[exp][n][\"train_info\"][\"best_epoch\"]                       # best epoch (restored)\n        ax.plot(b, h[\"val_loss\"][b - 1], \"o\", color=COLORS[n], ms=8, mec=\"white\", mew=2)\n    ax.set_yscale(\"log\")\n    ax.set_title(title, fontsize=12, loc=\"left\")\n    ax.set_xlabel(\"Epoch\")\n    ax.grid(True, which=\"major\", color=\"#e5e5e5\", lw=0.8)\n    ax.legend(frameon=False, fontsize=9)\n\naxes[0].set_ylabel(\"CTC loss (log scale)\")\nfig.text(0.5, -0.02, \"Solid = validation loss, dashed = training loss, \"\n         \"dot = best epoch (weights restored by early stopping).\", ha=\"center\", fontsize=9, color=\"#555555\")\nplt.tight_layout()\nplt.savefig(\"/kaggle/working/learning_curves.png\", dpi=200, bbox_inches=\"tight\")   # high resolution for the paper\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-13T14:16:17.910667Z","iopub.execute_input":"2026-10-13T14:16:17.911015Z","iopub.status.idle":"2026-10-13T14:16:19.628178Z","shell.execute_reply.started":"2026-10-13T14:16:17.910987Z","shell.execute_reply":"2026-10-13T14:16:19.627209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==== Cell 26: Error analysis - which characters does the model confuse? ====\nfrom collections import Counter\nfrom difflib import SequenceMatcher\n\ndef char_errors(true_list, pred_list):\n    \"\"\"Approximate character alignment (for understanding error TYPES; official metrics use edit distance).\"\"\"\n    subs, dels, ins = Counter(), Counter(), Counter()\n    for t, p in zip(true_list, pred_list):\n        for op, i1, i2, j1, j2 in SequenceMatcher(None, t, p, autojunk=False).get_opcodes():\n            if op == \"replace\" and (i2 - i1) == (j2 - j1):\n                for a, b in zip(t[i1:i2], p[j1:j2]):\n                    subs[f\"{a} -> {b}\"] += 1\n            else:\n                if op in (\"replace\", \"delete\"):\n                    for a in t[i1:i2]: dels[a] += 1\n                if op in (\"replace\", \"insert\"):\n                    for b in p[j1:j2]: ins[b] += 1\n    return subs, dels, ins\n\nrows = []\nfor exp, n in [(\"exp1\", \"DS1\"), (\"exp2\", \"DS2\"), (\"exp2\", \"DS3\")]:\n    s, d, i = char_errors(R[exp][n][\"test_labels\"], R[exp][n][\"test_preds\"])\n    print(f\"\\n===== {n} ({exp}) | substitutions = {sum(s.values())}, \"\n          f\"deletions = {sum(d.values())}, insertions = {sum(i.values())} =====\")\n    print(\"  Most common substitutions:\", s.most_common(8))\n    print(\"  Most often deleted chars  :\", d.most_common(5))\n    for k, c in s.most_common(10):\n        rows.append({\"Experiment\": exp, \"Dataset\": n, \"Substitution (true -> predicted)\": k, \"Count\": c})\n\npd.DataFrame(rows).to_csv(\"/kaggle/working/top_confusions.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-13T14:19:26.480107Z","iopub.execute_input":"2026-10-13T14:19:26.480839Z","iopub.status.idle":"2026-10-13T14:19:26.534023Z","shell.execute_reply.started":"2026-10-13T14:19:26.480802Z","shell.execute_reply":"2026-10-13T14:19:26.532797Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## Notes on how this notebook was run\n\n- **Cells 1–23** were run with a **GPU** (Kaggle T4). They prepare the data, build the model, train and evaluate it, and save all results to `results_all.json` (Cell 23).\n- **Cells 24–26** run on **CPU only** and do **not** retrain the model. They load the saved results (dataset *CAPTCHA Security Evaluation Results v2*) to build the results tables, the learning curves and the error analysis. This avoids retraining, saves GPU time and avoids waiting in the GPU queue.\n- The execution numbers `[n]` restart from 1 in every new session, so they are not sequential across sessions.\n\n### Why there are two experiments\n- **Experiment 1** was run as planned in the methodology (1,000 images per dataset). DS1 was broken, while DS2 and DS3 were not broken. The learning curves show that, with only 700 training images, the model could not learn to read DS2 and DS3.\n- **Experiment 2** was therefore added for DS2 and DS3, with 10,000 images each. It uses **exactly the same model, preprocessing, split ratio, training settings and metrics**; only the number of images changed. Both datasets were broken, which shows that their resistance in Experiment 1 was mainly caused by the limited training data. DS1 is not included because it contains only 1,070 images (and it was already broken in Experiment 1).\n- The results of the two experiments are not compared with each other, because their training sizes differ.\n\n### Official results (Version 2)\n| Experiment | Dataset | Character Accuracy | CAPTCHA-level Accuracy | Broken (≥ 1%)? |\n|---|---|---|---|---|\n| 1 | DS1 | 98.67% | 93.33% (140/150) | Yes |\n| 1 | DS2 | 4.53% | 0.00% (0/150) | No |\n| 1 | DS3 | 5.60% | 0.00% (0/150) | No |\n| 2 | DS2 | 95.65% | 80.47% (1207/1500) | Yes |\n| 2 | DS3 | 95.80% | 81.40% (1221/1500) | Yes |\n\n- In Experiment 2, the difference between DS2 and DS3 is **not statistically significant** (z = 0.65, p = 0.515).\n- The official results are those of **Version 2**. Retraining on a GPU can give slightly different numbers, so the models are not retrained in later versions.","metadata":{}},{"cell_type":"code","source":"# ==== Cell 27: Additional analyses (per-character error rates, Fisher test, 95% CI) ====\n# Uses only the saved official results (results_all.json); no training is needed.\nimport json, os, math, difflib\nfrom collections import Counter\nimport matplotlib.pyplot as plt\nfrom scipy.stats import fisher_exact\n\nPATH = \"/kaggle/input/datasets/banderma11/captcha-security-evaluation-results-v2/results_all.json\"\nif not os.path.exists(PATH):          # fallback for running outside Kaggle\n    PATH = \"results_all.json\"\nwith open(PATH) as f:\n    R = json.load(f)\n\n# ---- 1) 95% confidence interval of CAPTCHA-level Accuracy (normal approximation) ----\ndef ci95(correct, n):\n    p = correct / n\n    return 100 * p, 100 * 1.96 * math.sqrt(p * (1 - p) / n)\n\nprint(\"95% confidence intervals (CAPTCHA-level Accuracy):\")\nfor exp, n in [(\"exp1\", 150), (\"exp2\", 1500)]:\n    for ds in R[exp]:\n        acc, half = ci95(R[exp][ds][\"correct\"], n)\n        print(f\"  {exp} {ds}: {acc:.2f}% ± {half:.2f}\")\n\n# ---- 2) Fisher's exact test, Experiment 1 (DS1 vs DS2, DS1 vs DS3) ----\nprint(\"\\nFisher's exact test, Experiment 1 (correct vs incorrect out of 150):\")\nc1 = R[\"exp1\"][\"DS1\"][\"correct\"]\nfor ds in [\"DS2\", \"DS3\"]:\n    c = R[\"exp1\"][ds][\"correct\"]\n    _, p = fisher_exact([[c1, 150 - c1], [c, 150 - c]])\n    print(f\"  DS1 ({c1}/150) vs {ds} ({c}/150): p = {p:.2e}\")\n\n# ---- 3) Per-character error rate, Experiment 2 ----\n# Same alignment rule as Cell 26: equal-length replace = substitutions,\n# unequal-length replace = deletions + insertions. A ground-truth character counts\n# as an error when it is substituted or deleted.\ndef char_errors(labels, preds):\n    err, total = Counter(), Counter()\n    for t, p in zip(labels, preds):\n        total.update(t)\n        for op, i1, i2, j1, j2 in difflib.SequenceMatcher(None, t, p).get_opcodes():\n            if op in (\"replace\", \"delete\"):\n                err.update(t[i1:i2])\n    return err, total\n\nrates = {}\nfor ds in [\"DS2\", \"DS3\"]:\n    err, total = char_errors(R[\"exp2\"][ds][\"test_labels\"], R[\"exp2\"][ds][\"test_preds\"])\n    rates[ds] = {c: (err[c], total[c], 100 * err[c] / total[c]) for c in total}\n\nwith open(\"char_error_rates.csv\", \"w\") as f:\n    f.write(\"Dataset,Character,Errors,Occurrences,Error rate (%)\\n\")\n    for ds in rates:\n        for c, (e, n, r) in sorted(rates[ds].items(), key=lambda x: -x[1][2]):\n            f.write(f\"{ds},{c},{e},{n},{r:.1f}\\n\")\n\nTOP = 8\nnames  = {\"DS2\": \"Break the CAPTCHA Challenge (one font)\", \"DS3\": \"Synthetic CAPTCHA Dataset (three fonts)\"}\ncolors = {\"DS2\": \"#eb6834\", \"DS3\": \"#1baf7a\"}\nfig, axes = plt.subplots(1, 2, figsize=(10, 3.6), sharex=True)\nfor ax, ds in zip(axes, [\"DS2\", \"DS3\"]):\n    top = sorted(rates[ds].items(), key=lambda x: -x[1][2])[:TOP][::-1]\n    labels = [c for c, _ in top]\n    values = [v[2] for _, v in top]\n    ax.barh(labels, values, color=colors[ds])\n    for y, (c, (e, n, r)) in enumerate(top):\n        ax.text(r + 0.6, y, f\"{r:.1f}% ({e}/{n})\", va=\"center\", fontsize=8)\n    ax.set_title(names[ds], fontsize=10)\n    ax.set_xlabel(\"Error rate (%)\")\n    ax.set_xlim(0, 48)\n    ax.spines[[\"top\", \"right\"]].set_visible(False)\naxes[0].set_ylabel(\"Ground-truth character\")\nplt.tight_layout()\nplt.savefig(\"char_error_rates.png\", dpi=200)\nplt.show()\n\nprint(\"\\nTop characters by error rate (Experiment 2):\")\nfor ds in rates:\n    top = sorted(rates[ds].items(), key=lambda x: -x[1][2])[:5]\n    print(\" \", ds, \", \".join(f\"{c}: {r:.1f}% ({e}/{n})\" for c, (e, n, r) in top))\nprint(\"\\nSaved: char_error_rates.png, char_error_rates.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T10:35:54.839691Z","iopub.execute_input":"2026-10-20T10:35:54.839916Z","iopub.status.idle":"2026-10-20T10:35:57.364384Z","shell.execute_reply.started":"2026-10-20T10:35:54.839891Z","shell.execute_reply":"2026-10-20T10:35:57.36345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}