{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":14597806,"sourceType":"datasetVersion","datasetId":9324481},{"sourceId":14598646,"sourceType":"datasetVersion","datasetId":9324957},{"sourceId":61309,"sourceType":"modelInstanceVersion","modelInstanceId":51253,"modelId":70442},{"sourceId":633041,"sourceType":"modelInstanceVersion","modelInstanceId":477262,"modelId":493199},{"sourceId":665897,"sourceType":"modelInstanceVersion","modelInstanceId":504026,"modelId":519087}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 🫀 SandevistanLR — ECG-Inspired Learning-Rate Schedules\n\nI’m sharing **SandevistanLR**, an ECG-inspired learning-rate scheduler that models optimization dynamics using **P–QRS–T cardiac waveforms**, rather than cosine or triangular cycles.\n\nThe motivation is to explore whether **physiologically inspired schedules** can introduce more structured phases of exploration, recovery, and restart during training.\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F5646883%2F1100a0b070396170dfb088201c832c16%2FSinusRhythmLabels.svg.png?generation=1769571813406221&alt=media)\n\n---\n\n### 🔬 Current Demos (Kaggle)\n\nInitial experiments were run on **CSIRO – Image2Biomass Prediction**, using both frameworks:\n\n- **TensorFlow / Keras demo:**  \n  https://www.kaggle.com/code/bachaboos/sandevistanlr-tensorflow-demo\n\n- **PyTorch demo:**  \n  https://www.kaggle.com/code/bachaboos/sandevistanlr-pytorch-demo\n\nThese notebooks focus on **correct per-step LR control**, logging, and validation rather than final benchmark scores.\n\n---\n\n### 📦 GitHub (code + docs)\n\n⚠️ **Note:** This project is **under active development**.  \nThe repository may contain incomplete experiments or evolving APIs.\n\nImplementation, adapters, and documentation:  \nhttps://github.com/takfarine/SandevistanLR\n\nFull demo notebooks are also included in:\n- `code_examples/SandevistanLR-PyTorch-Demo.ipynb`\n- `code_examples/SandevistanLR-TensorFlow-Demo.ipynb`\n\n---\n\n### 📄 Research Direction\n\nThe long-term goal is to develop this into a **research-grade study** suitable for submission to **CVPR, NeurIPS, or ICLR** (workshops or main tracks, depending on empirical scope).\n\nPlanned directions include:\n- ablation of ECG components (P wave, QRS complex, T wave)\n- comparisons against cosine, 1cycle, and warm-restart baselines\n- stability, recovery, and convergence analysis across tasks\n\n---\n\n### 🤝 Research Collaboration & Academic / Industry Alignment\n\nI’m sharing this work with the intention of engaging with researchers and practitioners working on **optimization, training dynamics, and large-scale deep learning systems**.\n\nI’m actively looking for:\n- a **co-author** to jointly develop this project through rigorous experimentation and analysis  \n- a **research laboratory or industry research group** where I can contribute as an **AI Research Scientist** or **Data Scientist & AI Researcher**\n\nI intend to develop this work first in an industry research context, or in parallel with industry work, with a possible continuation within a PhD program at a later stage.\n\n\n---\n\n### 🚧 Ongoing & Future Work\n\nIn parallel, I’m developing a **second learning-rate project** exploring a different class of adaptive scheduling dynamics.  \nBoth projects are intended to mature into **complementary research papers**.\n\n---\n\nIf you’re interested in discussing collaboration, co-authorship, or related research directions, feel free to reach out via  \n**LinkedIn:** https://www.linkedin.com/in/takfarines-guergueb-b898b2218/  \nor **Email:** takfarinesguergueb@gmail.com\n\n","metadata":{}},{"cell_type":"code","source":"!pip install sandevistanlr","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# CSIRO Image2Biomass — Pure PyTorch + SandevistanLR + Validation\n# ============================================================\nfrom sandevistanlr import SandevistanScheduleBase\nfrom sandevistanlr.adapters.pytorch import SandevistanLR\n\nimport os, gc, re\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nimport csv\nfrom datetime import datetime\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nimport timm\nfrom safetensors.torch import load_file\nfrom sklearn.model_selection import GroupKFold\n\n\n# ----------------------------\n# CONFIG\n# ----------------------------\nSEED = 999\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\n\nDATA_ROOT = \"/kaggle/input/csiro-biomass\"\nWORK_DIR  = \"/kaggle/working\"\nPath(WORK_DIR).mkdir(parents=True, exist_ok=True)\n\nIMG_H, IMG_W = 256, 512\nCROP = 256\n\nBATCH_SIZE = 2\nEPOCHS     = 3\n\nLR_MAX = 2e-5\nLR_MIN = LR_MAX / 50\n\nTARGETS = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\nWEIGHTS = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5], dtype=torch.float32)\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"DEVICE:\", DEVICE)\n\nLOG_PATH = f\"{WORK_DIR}/training_log.csv\"\n\nwith open(LOG_PATH, \"w\", newline=\"\") as f:\n    csv.writer(f).writerow([\n        \"timestamp\", \"epoch\", \"step\", \"global_step\", \"lr\", \"loss\"\n    ])\n\n\n# ============================================================\n# DATA LOADING\n# ============================================================\ntrain_long = pd.read_csv(f\"{DATA_ROOT}/train.csv\")\n\ntrain_wide = train_long.pivot_table(\n    index=\"image_path\",\n    columns=\"target_name\",\n    values=\"target\",\n    aggfunc=\"first\"\n).reset_index()\n\nfor t in TARGETS:\n    if t not in train_wide.columns:\n        train_wide[t] = 0.0\n\ntrain_wide[TARGETS] = (\n    np.log1p(train_wide[TARGETS].fillna(0.0))\n    .astype(\"float32\")\n)\n\ntrain_wide[\"path\"] = train_wide[\"image_path\"].apply(\n    lambda p: p if str(p).startswith(\"/\") else os.path.join(DATA_ROOT, str(p))\n)\n\n\n# ============================================================\n# ROBUST GROUP DETECTION\n# ============================================================\ndef find_col(df, keys):\n    for c in df.columns:\n        for k in keys:\n            if k in c.lower():\n                return c\n    return None\n\ndate_col = find_col(train_long, [\"date\", \"sampling\"])\nsite_col = find_col(train_long, [\"site\", \"farm\", \"location\", \"property\", \"plot\"])\n\nmeta = train_long[[\"image_path\"]].drop_duplicates(\"image_path\").copy()\n\nif date_col:\n    dt = pd.to_datetime(train_long[date_col], errors=\"coerce\")\n    meta[\"_month\"] = dt.dt.to_period(\"M\").astype(str)\nelse:\n    meta[\"_month\"] = \"UNKMONTH\"\n\nif site_col:\n    meta[\"_site\"] = train_long[site_col].astype(str)\nelse:\n    meta[\"_site\"] = \"UNKSITE\"\n\nmeta[\"group\"] = meta[\"_month\"] + \"__\" + meta[\"_site\"]\n\ntrain_wide = train_wide.merge(\n    meta[[\"image_path\", \"group\"]],\n    on=\"image_path\",\n    how=\"left\"\n)\ntrain_wide[\"group\"] = train_wide[\"group\"].fillna(\"UNKGROUP\")\n\ngkf = GroupKFold(n_splits=5)\ntrain_idx, val_idx = next(gkf.split(train_wide, groups=train_wide[\"group\"]))\n\ntrain_df = train_wide.iloc[train_idx].reset_index(drop=True)\nval_df   = train_wide.iloc[val_idx].reset_index(drop=True)\n\n\n# ============================================================\n# DATASETS\n# ============================================================\ntrain_tfms = transforms.Compose([\n    transforms.Resize((IMG_H, IMG_W)),\n    transforms.RandomCrop(CROP),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225]),\n])\n\nval_tfms = transforms.Compose([\n    transforms.Resize((IMG_H, IMG_W)),\n    transforms.CenterCrop(CROP),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225]),\n])\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, tfms):\n        self.df = df.reset_index(drop=True)\n        self.tfms = tfms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        img = Image.open(self.df.loc[i, \"path\"]).convert(\"RGB\")\n        x = self.tfms(img)\n\n        y = torch.from_numpy(\n            self.df.loc[i, TARGETS].astype(\"float32\").to_numpy()\n        )\n        return x, y\n\n\ntrain_loader = DataLoader(\n    BiomassDataset(train_df, train_tfms),\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    num_workers=2,\n    pin_memory=True\n)\n\nval_loader = DataLoader(\n    BiomassDataset(val_df, val_tfms),\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=2,\n    pin_memory=True\n)\n\n\n# ============================================================\n# MODEL\n# ============================================================\nmodel = timm.create_model(\"convnextv2_base\", pretrained=False)\nsd = load_file(\"/kaggle/input/nconvnextv2-base/model (1).safetensors\")\nmodel.load_state_dict(sd, strict=False)\nmodel.reset_classifier(len(TARGETS))\nmodel.to(DEVICE)\n\n\n# ============================================================\n# LOSS + OPTIMIZER\n# ============================================================\nclass WeightedMSE(nn.Module):\n    def __init__(self, w):\n        super().__init__()\n        self.register_buffer(\"w\", w.view(1, -1))\n    def forward(self, p, y):\n        return ((p - y) ** 2 * self.w).mean()\n\ncriterion = WeightedMSE(WEIGHTS.to(DEVICE))\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR_MAX)\n\n\n# ============================================================\n# SANDEVISTANLR \n# ============================================================\nbase_schedule = SandevistanScheduleBase(\n    steps_per_epoch=len(train_loader),\n    epoch_plan=[\n        (\"normal\", 1),\n        (\"run\", 8),\n        (\"normal\", 1),\n        (\"run\", 10),\n        (\"die\", 0),\n        (\"aed\", 2),\n        (\"normal\", 1),\n    ],\n    lr_min=LR_MIN,\n    lr_max=LR_MAX,\n)\n\nschedule = SandevistanLR(base_schedule)\n\n\n# ============================================================\n# VALIDATION\n# ============================================================\n@torch.no_grad()\ndef validate(model, loader):\n    model.eval()\n    ps, ys = [], []\n    for x, y in loader:\n        x, y = x.to(DEVICE), y.to(DEVICE)\n        p = model(x)\n        ps.append(torch.expm1(p))\n        ys.append(torch.expm1(y))\n    p = torch.cat(ps)\n    y = torch.cat(ys)\n    w = WEIGHTS.to(p.device).view(1, -1)\n    mu = y.mean(0, keepdim=True)\n    ss_res = (w * (p - y) ** 2).sum()\n    ss_tot = (w * (y - mu) ** 2).sum()\n    return (1 - ss_res / ss_tot).item()\n\n\n# ============================================================\n# TRAIN LOOP\n# ============================================================\nglobal_step = 0\nbest_r2 = -1e9\n\nfor epoch in range(EPOCHS):\n    model.train()\n    pbar = tqdm(train_loader, desc=f\"Epoch {epoch}/{EPOCHS-1}\")\n\n    for step, (x, y) in enumerate(pbar):\n        x, y = x.to(DEVICE), y.to(DEVICE)\n\n        lr = schedule(global_step)\n        for pg in optimizer.param_groups:\n            pg[\"lr\"] = lr\n\n        optimizer.zero_grad(set_to_none=True)\n        loss = criterion(model(x), y)\n        loss.backward()\n        optimizer.step()\n\n        pbar.set_postfix(\n            step=step,\n            gstep=global_step,\n            lr=f\"{lr:.2e}\",\n            loss=f\"{loss.item():.4f}\",\n        )\n\n        with open(LOG_PATH, \"a\", newline=\"\") as f:\n            csv.writer(f).writerow([\n                datetime.utcnow().isoformat(),\n                epoch, step, global_step, lr, loss.item()\n            ])\n\n        global_step += 1\n\n    val_r2 = validate(model, val_loader)\n    print(f\"Epoch {epoch} | VAL weighted_r2 = {val_r2:.5f}\")\n\n    if val_r2 > best_r2:\n        best_r2 = val_r2\n        torch.save(model.state_dict(), f\"{WORK_DIR}/best_model.pt\")\n        print(\"✅ Saved best model\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:55:20.060447Z","iopub.execute_input":"2026-01-27T15:55:20.061501Z","execution_failed":"2026-01-27T15:55:33.636Z"}},"outputs":[],"execution_count":null}]}