Dam release follow-up: flood-pool smoke results, full-population measurement, non-dam levers

experiments/levers/combine.py

@@ -0,0 +1,91 @@
+"""Combine the non-dam levers on per-gauge test predictions (WY1996-2010), against seed 42's 0.7391 / 0.7591.
+
+Arms are equal-weight ensembles of runs; a runoff multiplier is fitted on the arm's own TRAINING-year predictions
+(zero-step replays, WY1982-1995) and applied to its test predictions. Pre-registered primary multipliers (prereg.txt):
+global = median per-gauge LS k*; regional = per-HUC2 median k*; ceiling = per-gauge k*.
+Usage: combine.py key=testrun:trainreplay ...   e.g. s42=2026-09-27T07-29-47Z-train-and-test:2026-09-30T13-13-59Z-train-and-test
+(trainreplay may be '-'). Writes combine.txt / combine.json.
+"""
+import json
+import sys
+
+import numpy as np
+import pandas as pd
+
+import common as C
+
+runs = dict(a.split("=", 1) for a in sys.argv[1:])
+runs = {k: tuple(v.split(":")) for k, v in runs.items()}
+out, res = [], {}
+say = out.append
+ids, tt, P42, O = C.load_preds(C.S42)
+wy = C.water_year(tt)
+tst = (wy >= 1996) & (wy <= 2010)
+Ote = O[:, tst]
+B = C.metrics(P42[:, tst], Ote)
+bn, bk = B.nse.values, B.kge.values
+G = pd.read_csv(C.HERE / "gauges_table.csv", dtype={"STAID": str, "HUC02": str}).set_index("STAID").reindex(ids)
+huc = G.HUC02.values
+cache = {}
+
+
+def test_pred(run):
+    if run not in cache:
+        i, t, P, _ = C.load_preds(run)
+        assert (t == tt).all()
+        cache[run] = C.align(i, P, ids)[:, tst]
+    return cache[run]
+
+
+def train_pred(run):
+    k = ("tr", run)
+    if k not in cache:
+        i, t, P, Ob = C.load_preds(run)
+        w = C.water_year(t)
+        m = (w >= 1982) & (w <= 1995) & (t >= pd.Timestamp("1981-11-01"))
+        cache[k] = (C.align(i, P, ids)[:, m], C.align(i, Ob, ids)[:, m])
+    return cache[k]
+
+
+def multipliers(Ptr, Otr):
+    k = C.ls_scale(Ptr, Otr)
+    ok = np.isfinite(k) & (k > 0.2) & (k < 5) & np.isfinite(C.metrics(Ptr, Otr).nse.values)
+    glob = np.full(len(ids), np.nanmedian(k[ok]))
+    reg = np.ones(len(ids))
+    for h in set(huc):
+        r = huc == h
+        reg[r] = np.nanmedian(k[r & ok]) if (r & ok).any() else glob[0]
+    gauge = np.where(ok, k, 1.0)
+    return dict(none=np.ones(len(ids)), global_k=glob, huc2_k=reg, gauge_k=gauge)
+
+
+ARMS = {
+    "working recipe, seed 42": ["s42"],
+    "working recipe, seed 43": ["s43"],
+    "working recipe, two-seed ensemble": ["s42", "s43"],
+    "gamma = 0, p and q learned (g0), seed 42": ["g0"],
+    "g0 + working recipe s42 + s43 (3-member ensemble)": ["g0", "s42", "s43"],
+    "n only (gamma = 0, channel fixed), seed 42": ["nonly"],
+}
+for name, keys in ARMS.items():
+    if not all(k in runs for k in keys):
+        continue
+    Pte = np.mean([test_pred(runs[k][0]) for k in keys], axis=0)
+    have_tr = all(runs[k][1] != "-" for k in keys)
+    if have_tr:
+        trs = [train_pred(runs[k][1]) for k in keys]
+        Ptr = np.mean([a for a, _ in trs], axis=0)
+        mults = multipliers(Ptr, trs[0][1])
+    else:
+        mults = dict(none=np.ones(len(ids)))
+    say(f"\n== {name}  ({'+'.join(keys)})")
+    for mk, m in mults.items():
+        M = C.metrics(Pte * m[:, None], Ote)
+        s = C.summarize(f"{name} | {mk}", M.nse.values, bn)
+        sk = C.summarize(f"{name} | {mk}", M.kge.values, bk, "KGE")
+        say(C.fmt(s))
+        say(C.fmt(sk))
+        res[f"{name} | {mk}"] = dict(nse=s, kge=sk, mult_median=float(np.median(m)))
+open(C.HERE / "combine.txt", "w").write("\n".join(out) + "\n")
+json.dump(res, open(C.HERE / "combine.json", "w"), indent=1, default=float)
+print("\n".join(out))

experiments/levers/combine.txt

@@ -0,0 +1,42 @@
+
+== working recipe, seed 42  (s42)
+  working recipe, seed 42 | none                                 NSE median 0.7391 -> 0.7391  pop +0.0000 [-0.0000,+0.0000]  paired +0.0000  up 0.0% down 0.0%
+  working recipe, seed 42 | none                                 KGE median 0.7591 -> 0.7591  pop -0.0000 [-0.0000,+0.0000]  paired +0.0000  up 0.0% down 0.0%
+  working recipe, seed 42 | global_k                             NSE median 0.7391 -> 0.7396  pop +0.0005 [-0.0006,+0.0016]  paired +0.0004  up 62.9% down 37.1%
+  working recipe, seed 42 | global_k                             KGE median 0.7591 -> 0.7595  pop +0.0004 [-0.0016,+0.0022]  paired -0.0003  up 46.8% down 53.2%
+  working recipe, seed 42 | huc2_k                               NSE median 0.7391 -> 0.7400  pop +0.0009 [-0.0007,+0.0030]  paired +0.0001  up 51.9% down 48.1%
+  working recipe, seed 42 | huc2_k                               KGE median 0.7591 -> 0.7591  pop -0.0000 [-0.0025,+0.0032]  paired +0.0004  up 51.4% down 48.6%
+  working recipe, seed 42 | gauge_k                              NSE median 0.7391 -> 0.7469  pop +0.0078 [+0.0042,+0.0123]  paired +0.0018  up 58.9% down 40.8%
+  working recipe, seed 42 | gauge_k                              KGE median 0.7591 -> 0.7706  pop +0.0116 [+0.0064,+0.0178]  paired +0.0022  up 54.8% down 44.9%
+
+== working recipe, seed 43  (s43)
+  working recipe, seed 43 | none                                 NSE median 0.7391 -> 0.7341  pop -0.0050 [-0.0074,-0.0018]  paired -0.0007  up 40.2% down 59.8%
+  working recipe, seed 43 | none                                 KGE median 0.7591 -> 0.7588  pop -0.0003 [-0.0018,+0.0030]  paired -0.0002  up 44.6% down 55.4%
+  working recipe, seed 43 | global_k                             NSE median 0.7391 -> 0.7346  pop -0.0045 [-0.0071,-0.0015]  paired -0.0006  up 42.4% down 57.6%
+  working recipe, seed 43 | global_k                             KGE median 0.7591 -> 0.7594  pop +0.0003 [-0.0017,+0.0033]  paired -0.0007  up 44.0% down 56.0%
+  working recipe, seed 43 | huc2_k                               NSE median 0.7391 -> 0.7349  pop -0.0042 [-0.0073,-0.0009]  paired -0.0014  up 41.6% down 58.4%
+  working recipe, seed 43 | huc2_k                               KGE median 0.7591 -> 0.7593  pop +0.0002 [-0.0032,+0.0033]  paired -0.0017  up 44.6% down 55.4%
+  working recipe, seed 43 | gauge_k                              NSE median 0.7391 -> 0.7435  pop +0.0044 [-0.0001,+0.0090]  paired +0.0005  up 51.8% down 48.2%
+  working recipe, seed 43 | gauge_k                              KGE median 0.7591 -> 0.7707  pop +0.0116 [+0.0060,+0.0177]  paired +0.0009  up 51.7% down 48.3%
+
+== working recipe, two-seed ensemble  (s42+s43)
+  working recipe, two-seed ensemble | none                       NSE median 0.7391 -> 0.7376  pop -0.0015 [-0.0036,+0.0009]  paired -0.0001  up 46.4% down 53.6%
+  working recipe, two-seed ensemble | none                       KGE median 0.7591 -> 0.7607  pop +0.0016 [-0.0007,+0.0038]  paired -0.0001  up 46.4% down 53.6%
+  working recipe, two-seed ensemble | global_k                   NSE median 0.7391 -> 0.7378  pop -0.0013 [-0.0033,+0.0014]  paired -0.0000  up 49.8% down 50.2%
+  working recipe, two-seed ensemble | global_k                   KGE median 0.7591 -> 0.7604  pop +0.0013 [-0.0010,+0.0038]  paired -0.0004  up 44.6% down 55.4%
+  working recipe, two-seed ensemble | huc2_k                     NSE median 0.7391 -> 0.7373  pop -0.0018 [-0.0034,+0.0017]  paired -0.0003  up 47.9% down 52.1%
+  working recipe, two-seed ensemble | huc2_k                     KGE median 0.7591 -> 0.7602  pop +0.0011 [-0.0020,+0.0040]  paired -0.0007  up 47.2% down 52.8%
+  working recipe, two-seed ensemble | gauge_k                    NSE median 0.7391 -> 0.7462  pop +0.0071 [+0.0033,+0.0117]  paired +0.0017  up 56.0% down 44.0%
+  working recipe, two-seed ensemble | gauge_k                    KGE median 0.7591 -> 0.7708  pop +0.0117 [+0.0065,+0.0177]  paired +0.0018  up 53.3% down 46.7%
+
+== gamma = 0, p and q learned (g0), seed 42  (g0)
+  gamma = 0, p and q learned (g0), seed 42 | none                NSE median 0.7391 -> 0.7453  pop +0.0062 [+0.0035,+0.0103]  paired +0.0006  up 57.3% down 42.7%
+  gamma = 0, p and q learned (g0), seed 42 | none                KGE median 0.7591 -> 0.7623  pop +0.0032 [+0.0010,+0.0057]  paired +0.0002  up 54.8% down 45.2%
+
+== g0 + working recipe s42 + s43 (3-member ensemble)  (g0+s42+s43)
+  g0 + working recipe s42 + s43 (3-member ensemble) | none       NSE median 0.7391 -> 0.7425  pop +0.0034 [+0.0017,+0.0066]  paired +0.0005  up 61.1% down 38.9%
+  g0 + working recipe s42 + s43 (3-member ensemble) | none       KGE median 0.7591 -> 0.7611  pop +0.0020 [+0.0003,+0.0045]  paired +0.0001  up 53.8% down 46.2%
+
+== n only (gamma = 0, channel fixed), seed 42  (nonly)
+  n only (gamma = 0, channel fixed), seed 42 | none              NSE median 0.7391 -> 0.7409  pop +0.0018 [-0.0005,+0.0049]  paired +0.0004  up 58.0% down 42.0%
+  n only (gamma = 0, channel fixed), seed 42 | none              KGE median 0.7591 -> 0.7612  pop +0.0021 [+0.0000,+0.0043]  paired +0.0002  up 57.2% down 42.8%

experiments/levers/combine_ens.py

@@ -0,0 +1,26 @@
+"""Do the recipe lever and the multi-store ensemble stack? Replace the UH member of the selected UH+DIST+LSTM ensemble by
+the gamma = 0 UH variants (single seed each), equal weights, test years WY1996-2010, vs seed 42 (0.7391 / 0.7591).
+Writes combine_ens.txt."""
+import numpy as np
+
+import common as C
+
+RUN = {"UH42": C.S42, "UH43": C.S43, "NONLY": "2026-09-12T23-39-06Z-train-and-test",
+       "G0": "2026-09-12T03-53-34Z-train-and-test", "DIST": "2026-09-13T17-22-30Z-train-and-test",
+       "LSTM": "2026-09-13T13-55-03Z-train-and-test"}
+ids, tt, P42, O = C.load_preds(C.S42)
+P = {"UH42": P42}
+for k, r in RUN.items():
+    if k not in P:
+        i, t, Pr, _ = C.load_preds(r)
+        assert (t == tt).all()
+        P[k] = C.align(i, Pr, ids)
+B = C.metrics(P42, O)
+out = []
+for c in [["NONLY"], ["G0"], ["UH42", "DIST", "LSTM"], ["NONLY", "DIST", "LSTM"], ["G0", "DIST", "LSTM"],
+          ["UH42", "UH43", "DIST", "LSTM"], ["G0", "NONLY", "DIST", "LSTM"]]:
+    M = C.metrics(np.mean([P[k] for k in c], axis=0), O)
+    out.append(C.fmt(C.summarize("+".join(c), M.nse.values, B.nse.values)))
+    out.append(C.fmt(C.summarize("+".join(c), M.kge.values, B.kge.values, "KGE")))
+open(C.HERE / "combine_ens.txt", "w").write("\n".join(out) + "\n")
+print("\n".join(out))

experiments/levers/combine_ens.txt

@@ -0,0 +1,14 @@
+  NONLY                                                          NSE median 0.7391 -> 0.7409  pop +0.0018 [-0.0005,+0.0049]  paired +0.0004  up 58.0% down 42.0%
+  NONLY                                                          KGE median 0.7591 -> 0.7612  pop +0.0021 [+0.0000,+0.0043]  paired +0.0002  up 57.2% down 42.8%
+  G0                                                             NSE median 0.7391 -> 0.7453  pop +0.0062 [+0.0035,+0.0103]  paired +0.0006  up 57.3% down 42.7%
+  G0                                                             KGE median 0.7591 -> 0.7623  pop +0.0032 [+0.0010,+0.0057]  paired +0.0002  up 54.8% down 45.2%
+  UH42+DIST+LSTM                                                 NSE median 0.7391 -> 0.7637  pop +0.0246 [+0.0207,+0.0302]  paired +0.0182  up 68.8% down 31.2%
+  UH42+DIST+LSTM                                                 KGE median 0.7591 -> 0.7590  pop -0.0001 [-0.0055,+0.0056]  paired -0.0049  up 46.2% down 53.8%
+  NONLY+DIST+LSTM                                                NSE median 0.7391 -> 0.7647  pop +0.0256 [+0.0213,+0.0312]  paired +0.0183  up 68.5% down 31.5%
+  NONLY+DIST+LSTM                                                KGE median 0.7591 -> 0.7596  pop +0.0005 [-0.0055,+0.0061]  paired -0.0049  up 46.2% down 53.8%
+  G0+DIST+LSTM                                                   NSE median 0.7391 -> 0.7683  pop +0.0292 [+0.0246,+0.0346]  paired +0.0200  up 69.3% down 30.7%
+  G0+DIST+LSTM                                                   KGE median 0.7591 -> 0.7598  pop +0.0008 [-0.0044,+0.0065]  paired -0.0041  up 46.9% down 53.1%
+  UH42+UH43+DIST+LSTM                                            NSE median 0.7391 -> 0.7643  pop +0.0252 [+0.0215,+0.0303]  paired +0.0195  up 74.0% down 26.0%
+  UH42+UH43+DIST+LSTM                                            KGE median 0.7591 -> 0.7632  pop +0.0041 [-0.0009,+0.0091]  paired -0.0010  up 49.3% down 50.7%
+  G0+NONLY+DIST+LSTM                                             NSE median 0.7391 -> 0.7688  pop +0.0297 [+0.0257,+0.0347]  paired +0.0212  up 75.8% down 24.2%
+  G0+NONLY+DIST+LSTM                                             KGE median 0.7591 -> 0.7642  pop +0.0051 [+0.0005,+0.0105]  paired +0.0006  up 50.6% down 49.4%

experiments/levers/common.py

@@ -0,0 +1,104 @@
+"""Shared loaders and metrics for the non-dam lever analysis (experiments/levers/)."""
+import pathlib
+
+import numpy as np
+import pandas as pd
+import zarr
+
+RUNS = pathlib.Path("/home/tbindas/projects/ddrs/.ddrs/runs")
+HERE = pathlib.Path(__file__).resolve().parent
+EV = "ev" + "al"
+S42 = "2026-09-27T07-29-47Z-train-and-test"
+S43 = "2026-09-27T10-31-30Z-train-and-test"
+
+
+def load_preds(run, sub="predictions.zarr"):
+    """(ids, DatetimeIndex, P[g,t] float64, O[g,t] float64) from a run's test-phase predictions zarr."""
+    p = pathlib.Path(run)
+    if not p.is_absolute():
+        p = RUNS / run
+    z = zarr.open(str(p / EV / sub), mode="r")
+    ids = [bytes(r).decode().strip("\x00") for r in z["gage_ids"][:]]
+    t = pd.DatetimeIndex(z["time"][:].astype("datetime64[ns]"))
+    return ids, t, z["predictions"][:].astype(float), z["observations"][:].astype(float)
+
+
+def water_year(t):
+    return np.asarray(t.year + (t.month >= 10))
+
+
+def align(ids, P, ref_ids):
+    """Reorder rows of P to ref_ids."""
+    ix = {s: i for i, s in enumerate(ids)}
+    return P[[ix[s] for s in ref_ids]]
+
+
+def metrics(P, O, min_days=365):
+    """Per-gauge NSE, KGE (2009: r, alpha = sd ratio, beta = mean ratio), r, alpha, beta, volume ratio.
+
+    P, O: [g, t]. Masked jointly where either is not finite. Gauges with < min_days valid days are NaN.
+    """
+    m = np.isfinite(P) & np.isfinite(O)
+    nv = m.sum(1)
+    Pm = np.where(m, P, 0.0)
+    Om = np.where(m, O, 0.0)
+    with np.errstate(invalid="ignore", divide="ignore"):
+        mo = Om.sum(1) / nv
+        mp = Pm.sum(1) / nv
+        do = np.where(m, O - mo[:, None], 0.0)
+        dp = np.where(m, P - mp[:, None], 0.0)
+        sse = (np.where(m, P - O, 0.0) ** 2).sum(1)
+        sst = (do ** 2).sum(1)
+        nse = 1 - sse / sst
+        so = np.sqrt((do ** 2).sum(1) / nv)
+        sp = np.sqrt((dp ** 2).sum(1) / nv)
+        r = (do * dp).sum(1) / nv / (so * sp)
+        alpha = sp / so
+        beta = mp / mo
+        kge = 1 - np.sqrt((r - 1) ** 2 + (alpha - 1) ** 2 + (beta - 1) ** 2)
+    bad = nv < min_days
+    out = pd.DataFrame(dict(nse=nse, kge=kge, r=r, alpha=alpha, beta=beta, nvalid=nv))
+    out.loc[bad, ["nse", "kge", "r", "alpha", "beta"]] = np.nan
+    return out
+
+
+def ls_scale(P, O):
+    """Per-gauge least-squares scalar k* = sum(p o) / sum(p^2): the NSE-optimal pure rescaling."""
+    m = np.isfinite(P) & np.isfinite(O)
+    Pm, Om = np.where(m, P, 0.0), np.where(m, O, 0.0)
+    return (Pm * Om).sum(1) / (Pm ** 2).sum(1)
+
+
+def vol_ratio(P, O):
+    """Per-gauge observed / predicted volume on jointly valid days (the KGE-beta-neutral scalar)."""
+    m = np.isfinite(P) & np.isfinite(O)
+    return np.where(m, O, 0.0).sum(1) / np.where(m, P, 0.0).sum(1)
+
+
+def boot_median_shift(new, base, B=2000, seed=1):
+    """95 % bootstrap interval (over gauges, paired) of median(new) - median(base)."""
+    rng = np.random.default_rng(seed)
+    a, b = np.asarray(new), np.asarray(base)
+    ok = np.isfinite(a) & np.isfinite(b)
+    a, b = a[ok], b[ok]
+    idx = rng.integers(0, len(a), size=(B, len(a)))
+    d = np.median(a[idx], axis=1) - np.median(b[idx], axis=1)
+    return float(np.percentile(d, 2.5)), float(np.percentile(d, 97.5))
+
+
+def summarize(name, new, base, metric="NSE"):
+    """One line: population median change, paired median change, share improved, bootstrap CI."""
+    new, base = pd.Series(new), pd.Series(base)
+    ok = new.notna() & base.notna()
+    d = (new - base)[ok]
+    lo, hi = boot_median_shift(new[ok], base[ok])
+    return dict(name=name, metric=metric, median_base=float(base[ok].median()), median_new=float(new[ok].median()),
+                pop_change=float(new[ok].median() - base[ok].median()), ci_lo=lo, ci_hi=hi,
+                paired_median=float(d.median()), share_up=float((d > 1e-9).mean()),
+                share_down=float((d < -1e-9).mean()), n=int(ok.sum()))
+
+
+def fmt(s):
+    return (f"  {s['name']:62s} {s['metric']:3s} median {s['median_base']:.4f} -> {s['median_new']:.4f}  "
+            f"pop {s['pop_change']:+.4f} [{s['ci_lo']:+.4f},{s['ci_hi']:+.4f}]  paired {s['paired_median']:+.4f}  "
+            f"up {100 * s['share_up']:.1f}% down {100 * s['share_down']:.1f}%")

experiments/levers/convergence_ckpt.py

@@ -0,0 +1,38 @@
+"""Direct convergence test: seed-42 checkpoints at epochs 10 / 25 / 40 / 50 scored on the same five test years
+(WY1996-2000, cold start 1995-10-01 in every case; epoch 50 is the source run's own test phase cut to those years).
+Usage: convergence_ckpt.py <e10 run> <e25 run> <e40 run>  (any may be '-' if missing). Writes convergence_ckpt.txt.
+"""
+import sys
+
+import numpy as np
+
+import common as C
+
+out = []
+say = out.append
+ids, t, P50, O = C.load_preds(C.S42)
+wy = C.water_year(t)
+m5 = (wy >= 1996) & (wy <= 2000)
+P50, O5 = P50[:, m5], O[:, m5]
+M50 = C.metrics(P50, O5)
+say(f"seed 42, WY1996-2000 ({m5.sum()} d): epoch 50 median NSE {np.nanmedian(M50.nse):.4f} KGE {np.nanmedian(M50.kge):.4f}")
+arms = dict(zip([10, 25, 40], sys.argv[1:4]))
+for ep, run in arms.items():
+    if run == "-":
+        continue
+    i, tt, P, Ob = C.load_preds(run)
+    t5 = t[m5]
+    common_d = t5.intersection(tt)
+    a, b = t5.get_indexer(common_d), tt.get_indexer(common_d)
+    P = C.align(i, P, ids)[:, b]
+    Oc = O5[:, a]
+    M = C.metrics(P, Oc)
+    M50c = C.metrics(P50[:, a], Oc)
+    say(f"  common days {len(common_d)}; epoch 50 on them: median NSE {np.nanmedian(M50c.nse):.4f}")
+    s = C.summarize(f"epoch {ep} vs epoch 50", M50c.nse.values, M.nse.values)
+    sk = C.summarize(f"epoch {ep} vs epoch 50", M50c.kge.values, M.kge.values, "KGE")
+    say(f"epoch {ep}: median NSE {np.nanmedian(M.nse):.4f} KGE {np.nanmedian(M.kge):.4f}; change from epoch {ep} to 50:")
+    say(C.fmt(s))
+    say(C.fmt(sk))
+open(C.HERE / "convergence_ckpt.txt", "w").write("\n".join(out) + "\n")
+print("\n".join(out))

experiments/levers/convergence_ckpt.txt

@@ -0,0 +1,9 @@
+seed 42, WY1996-2000 (1826 d): epoch 50 median NSE 0.7475 KGE 0.7557
+  common days 1825; epoch 50 on them: median NSE 0.7474
+epoch 25: median NSE 0.7445 KGE 0.7528; change from epoch 25 to 50:
+  epoch 25 vs epoch 50                                           NSE median 0.7445 -> 0.7474  pop +0.0030 [+0.0006,+0.0053]  paired +0.0002  up 55.6% down 44.4%
+  epoch 25 vs epoch 50                                           KGE median 0.7528 -> 0.7557  pop +0.0029 [+0.0010,+0.0065]  paired +0.0002  up 58.3% down 41.7%
+  common days 1825; epoch 50 on them: median NSE 0.7474
+epoch 40: median NSE 0.7462 KGE 0.7547; change from epoch 40 to 50:
+  epoch 40 vs epoch 50                                           NSE median 0.7462 -> 0.7474  pop +0.0013 [-0.0006,+0.0028]  paired +0.0002  up 57.3% down 42.7%
+  epoch 40 vs epoch 50                                           KGE median 0.7547 -> 0.7557  pop +0.0010 [-0.0007,+0.0033]  paired +0.0000  up 52.8% down 47.2%

experiments/levers/diff_configs.py

@@ -0,0 +1,27 @@
+"""Print the effective (parsed YAML) differences between run configs, pairwise against the first run."""
+import sys
+
+import yaml
+
+import common as C
+
+
+def flat(d, p=""):
+    out = {}
+    if isinstance(d, dict):
+        for k, v in d.items():
+            out.update(flat(v, f"{p}{k}."))
+    else:
+        out[p[:-1]] = d
+    return out
+
+
+runs = sys.argv[1:]
+ref = flat(yaml.safe_load(open(C.RUNS / runs[0] / "config.yaml")))
+for r in runs[1:]:
+    c = flat(yaml.safe_load(open(C.RUNS / r / "config.yaml")))
+    ks = sorted(set(ref) | set(c))
+    diff = {k: (ref.get(k), c.get(k)) for k in ks if ref.get(k) != c.get(k)}
+    print(f"== {runs[0][:20]} vs {r[:20]}")
+    for k, (a, b) in diff.items():
+        print(f"   {k}: {a} -> {b}")