{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"import os, json, shutil, zipfile\nfrom pathlib import Path\nfrom datetime import datetime\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nimport torch\nfrom fastai.vision.all import *\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Device: {device}\")\nif device.type == 'cuda':\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n# print(f\"fastai version: {fastai.__version__}\")\nprint(f\"torch version: {torch.__version__}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:09.984508Z","iopub.execute_input":"2026-10-15T14:08:09.984804Z","iopub.status.idle":"2026-10-15T14:08:25.734852Z","shell.execute_reply.started":"2026-10-15T14:08:09.984781Z","shell.execute_reply":"2026-10-15T14:08:25.734005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATASET_DIR = Path('/kaggle/input/datasets/utkarshtomar736/dvc-fastai-experiments')\nLOG_PATH = Path('/kaggle/working/experiments.csv')\nRUNS_DIR = Path('/kaggle/working/runs')\n\nif DATASET_DIR.exists():\n    # Restore experiments.csv\n    if (DATASET_DIR / 'experiments.csv').exists() and not LOG_PATH.exists():\n        shutil.copy(DATASET_DIR / 'experiments.csv', LOG_PATH)\n        print(\"Restored experiments.csv from dataset\")\n\n    # Restore runs folder\n    if (DATASET_DIR / 'runs.zip').exists() and not RUNS_DIR.exists():\n        shutil.unpack_archive(DATASET_DIR / 'runs.zip', '/kaggle/working/')\n        n = len(list(RUNS_DIR.iterdir()))\n        print(f\"Restored {n} prior runs from dataset\")\nelse:\n    print(\"No dataset attached — starting fresh log\")\n\nif LOG_PATH.exists():\n    print(\"\\nExisting experiments:\")\n    display(pd.read_csv(LOG_PATH))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:25.7363Z","iopub.execute_input":"2026-10-15T14:08:25.737311Z","iopub.status.idle":"2026-10-15T14:08:25.784512Z","shell.execute_reply.started":"2026-10-15T14:08:25.737277Z","shell.execute_reply":"2026-10-15T14:08:25.783958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COMP_DIR = Path('/kaggle/input/competitions/dogs-vs-cats-redux-kernels-edition')\nWORK_DIR = Path('/kaggle/working/dogscats')\nWORK_DIR.mkdir(exist_ok=True)\n\nif not (WORK_DIR / 'train').exists():\n    with zipfile.ZipFile(COMP_DIR / 'train.zip', 'r') as z:\n        z.extractall(WORK_DIR / 'train')\nif not (WORK_DIR / 'test').exists():\n    with zipfile.ZipFile(COMP_DIR / 'test.zip', 'r') as z:\n        z.extractall(WORK_DIR / 'test')\n\nprint(f\"Train images: {len(list((WORK_DIR/'train').glob('*.jpg')))}\")\nprint(f\"Test images:  {len(list((WORK_DIR/'test').glob('*.jpg')))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:25.785309Z","iopub.execute_input":"2026-10-15T14:08:25.786227Z","iopub.status.idle":"2026-10-15T14:08:37.452492Z","shell.execute_reply.started":"2026-10-15T14:08:25.78617Z","shell.execute_reply":"2026-10-15T14:08:37.45169Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## DataBlock with Augmentation","metadata":{}},{"cell_type":"code","source":"def label_func(x):\n    return 'cat' if x.name.startswith('cat') else 'dog'\n\ndblock = DataBlock(\n    blocks=(ImageBlock, CategoryBlock),\n    get_items=get_image_files,\n    splitter=RandomSplitter(valid_pct=0.2, seed=42),  # same split as baseline\n    get_y=label_func,\n    item_tfms=Resize(224),\n    batch_tfms=[\n        *aug_transforms(size=224),                      # <-- the change\n        Normalize.from_stats(*imagenet_stats),          # explicit normalization\n    ],\n)\n\ndls = dblock.dataloaders(WORK_DIR / 'train', bs=64)\nprint(f\"Train batches: {len(dls.train)}, Val batches: {len(dls.valid)}\")\nprint(f\"Classes: {dls.vocab}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:37.453458Z","iopub.execute_input":"2026-10-15T14:08:37.453764Z","iopub.status.idle":"2026-10-15T14:08:39.012865Z","shell.execute_reply.started":"2026-10-15T14:08:37.453735Z","shell.execute_reply":"2026-10-15T14:08:39.012221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dls.show_batch(max_n=6, nrows=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:39.014438Z","iopub.execute_input":"2026-10-15T14:08:39.014711Z","iopub.status.idle":"2026-10-15T14:08:39.934602Z","shell.execute_reply.started":"2026-10-15T14:08:39.014683Z","shell.execute_reply":"2026-10-15T14:08:39.93366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xb, yb = dls.one_batch()\nprint(f\"Batch shape: {xb.shape}, labels: {yb.shape}\")\nprint(f\"Pixel range: [{xb.min().item():.2f}, {xb.max().item():.2f}]\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:39.935831Z","iopub.execute_input":"2026-10-15T14:08:39.936186Z","iopub.status.idle":"2026-10-15T14:08:40.226288Z","shell.execute_reply.started":"2026-10-15T14:08:39.936143Z","shell.execute_reply":"2026-10-15T14:08:40.225621Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Experiment Tracking","metadata":{}},{"cell_type":"code","source":"RUNS_DIR = Path('/kaggle/working/runs')\nRUNS_DIR.mkdir(exist_ok=True)\nLOG_PATH = Path('/kaggle/working/experiments.csv')\n\ndef log_run(**kwargs):\n    row = {'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M'), **kwargs}\n    if LOG_PATH.exists():\n        df = pd.read_csv(LOG_PATH)\n        df = pd.concat([df, pd.DataFrame([row])], ignore_index=True)\n    else:\n        df = pd.DataFrame([row])\n    df.to_csv(LOG_PATH, index=False)\n    print(f\"Logged run. Total rows: {len(df)}\")\n    return df\n\ndef save_run_artifacts(learn, run_id, config):\n    run_dir = RUNS_DIR / run_id\n    run_dir.mkdir(parents=True, exist_ok=True)\n\n    try:\n        learn.recorder.plot_loss()\n        plt.savefig(run_dir / 'loss_curve.png', dpi=100, bbox_inches='tight')\n        plt.close()\n    except Exception as e:\n        print(f\"  loss_curve failed: {e}\")\n\n    try:\n        interp = ClassificationInterpretation.from_learner(learn)\n        interp.plot_confusion_matrix()\n        plt.savefig(run_dir / 'confusion_matrix.png', dpi=100, bbox_inches='tight')\n        plt.close()\n\n        interp.plot_top_losses(9, nrows=3)\n        plt.savefig(run_dir / 'top_losses.png', dpi=100, bbox_inches='tight')\n        plt.close()\n    except Exception as e:\n        print(f\"  interpretation failed: {e}\")\n\n    with open(run_dir / 'config.json', 'w') as f:\n        json.dump(config, f, indent=2, default=str)\n\n    print(f\"Artifacts saved to {run_dir}:\")\n    for p in sorted(run_dir.iterdir()):\n        print(f\"  {p.name}\")\n\nprint(\"Tracking setup ready.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:50.680464Z","iopub.execute_input":"2026-10-15T14:08:50.681257Z","iopub.status.idle":"2026-10-15T14:08:50.690836Z","shell.execute_reply.started":"2026-10-15T14:08:50.681221Z","shell.execute_reply":"2026-10-15T14:08:50.690207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"run_id = 'dvc_002'\n\nconfig = {\n    'run_id': run_id,\n    'description': 'baseline + aug_transforms + Normalize',\n    'arch': 'resnet34',\n    'img_size': 224,\n    'batch_size': 64,\n    'augmentation': 'aug_transforms (default)',\n    'lr': 'default (fine_tune)',\n    'epochs': 3,\n    'notes': 'one change from dvc_001: augmentation',\n}\n\nprint(json.dumps(config, indent=2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:08:56.344813Z","iopub.execute_input":"2026-10-15T14:08:56.345277Z","iopub.status.idle":"2026-10-15T14:08:56.350413Z","shell.execute_reply.started":"2026-10-15T14:08:56.345246Z","shell.execute_reply":"2026-10-15T14:08:56.349416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learn = vision_learner(\n    dls,\n    resnet34,\n    metrics=[accuracy, error_rate],\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:09:06.998062Z","iopub.execute_input":"2026-10-15T14:09:06.998549Z","iopub.status.idle":"2026-10-15T14:09:08.00186Z","shell.execute_reply.started":"2026-10-15T14:09:06.998519Z","shell.execute_reply":"2026-10-15T14:09:08.000981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Diagnostic only — look at the plot, note the suggested LRs, move on.\nlearn.lr_find()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:09:12.177338Z","iopub.execute_input":"2026-10-15T14:09:12.177794Z","iopub.status.idle":"2026-10-15T14:09:35.995305Z","shell.execute_reply.started":"2026-10-15T14:09:12.17776Z","shell.execute_reply":"2026-10-15T14:09:35.99445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learn.fine_tune(3, base_lr=0.00048)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:12:05.033201Z","iopub.execute_input":"2026-10-15T14:12:05.033893Z","iopub.status.idle":"2026-10-15T14:20:03.942577Z","shell.execute_reply.started":"2026-10-15T14:12:05.033845Z","shell.execute_reply":"2026-10-15T14:20:03.941715Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save Logs","metadata":{}},{"cell_type":"code","source":"val_loss, val_acc, val_err = learn.validate()\nprint(f\"val_loss={val_loss:.4f}  val_acc={val_acc:.4f}  val_err={val_err:.4f}\")\n\nsave_run_artifacts(learn, run_id, config)\n\nlog_run(\n    **{k: v for k, v in config.items() if k != 'run_id'},\n    run_id=run_id,\n    val_loss=round(float(val_loss), 4),\n    val_acc=round(float(val_acc), 4),\n    lb_score=None,\n)\n\n# Show the comparison table\ndisplay(pd.read_csv(LOG_PATH))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:20:32.708617Z","iopub.execute_input":"2026-10-15T14:20:32.709521Z","iopub.status.idle":"2026-10-15T14:21:08.519837Z","shell.execute_reply.started":"2026-10-15T14:20:32.709487Z","shell.execute_reply":"2026-10-15T14:21:08.518888Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"test_files = get_image_files(WORK_DIR / 'test').sorted()\ntest_dl = dls.test_dl(test_files, with_labels=False)\n\npreds, _ = learn.get_preds(dl=test_dl)\nprobs = preds[:, 1]\n\nsubmission = pd.read_csv(COMP_DIR / 'sample_submission.csv')\nsubmission['label'] = probs.numpy()\nsubmission.to_csv('/kaggle/working/submission_dvc002.csv', index=False)\n\nprint(submission.head())\nprint(f\"Prob range: [{submission['label'].min():.4f}, {submission['label'].max():.4f}]\")\nprint(f\"Mean prob: {submission['label'].mean():.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:21:23.705188Z","iopub.execute_input":"2026-10-15T14:21:23.705714Z","iopub.status.idle":"2026-10-15T14:21:49.972472Z","shell.execute_reply.started":"2026-10-15T14:21:23.705675Z","shell.execute_reply":"2026-10-15T14:21:49.971517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if RUNS_DIR.exists():\n    shutil.make_archive('/kaggle/working/runs', 'zip', RUNS_DIR)\n\nfrom IPython.display import FileLink\nprint(\"Download and re-upload to your dvc-experiments dataset as a new version:\")\nFileLink('/kaggle/working/experiments.csv')\nFileLink('/kaggle/working/runs.zip')\nFileLink('/kaggle/working/submission_dvc002.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-15T14:21:49.974084Z","iopub.execute_input":"2026-10-15T14:21:49.974409Z","iopub.status.idle":"2026-10-15T14:21:50.0113Z","shell.execute_reply.started":"2026-10-15T14:21:49.974378Z","shell.execute_reply":"2026-10-15T14:21:50.010607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}