Add LUCID to the CLI, inspect, skill, notebook and llms.txt

causalts/cli.py

@@ -1224,6 +1224,154 @@ def plot(ctx, graph, plot_type, val_matrix, var_names, figsize, plot_format, sav
     _log(ctx, f"Saved {fname} to {outdir}")
 
 
+# ---------------------------------------------------------------------------
+# deconfound — LUCID latent-confounder corrections on a discovered graph
+# ---------------------------------------------------------------------------
+@main.command()
+@click.argument("graph_path", metavar="GRAPH", type=click.Path(exists=True))
+@click.option(
+    "--data",
+    "data_path",
+    required=True,
+    type=click.Path(exists=True),
+    help="Data file the graph was discovered from (CSV/parquet/feather).",
+)
+@click.option(
+    "--strategy",
+    type=click.Choice(["adaptive", "tetrad", "pds"]),
+    default="adaptive",
+    show_default=True,
+    help=(
+        "adaptive = LUCID (infers the regime, applies the matching correction); "
+        "tetrad/pds = fixed single-strategy comparators."
+    ),
+)
+@click.option(
+    "--threshold",
+    type=float,
+    default=None,
+    help="Tetrad factor-consistency threshold (--strategy tetrad only). Default 0.25.",
+)
+@click.option(
+    "--alpha",
+    type=float,
+    default=None,
+    help="Significance level for the PDS filter (--strategy pds only). Default 1e-10.",
+)
+@click.option(
+    "--var-names", type=str, default=None, help="Comma-separated variable names."
+)
+@click.option(
+    "--json",
+    "output_json",
+    is_flag=True,
+    default=False,
+    help="Echo the run summary as JSON to stdout.",
+)
+@click.pass_context
+def deconfound(
+    ctx, graph_path, data_path, strategy, threshold, alpha, var_names, output_json
+):
+    """Correct a discovered graph for latent confounders.
+
+    GRAPH is an .npy file produced by `discover` (e.g. estimated_graph.npy); --data
+    is the series it was discovered from. max_lag is read from the graph's shape.
+
+    The default `adaptive` strategy is LUCID: it infers from the residual spectrum
+    whether latent confounding is sparse or pervasive and applies the matching
+    correction. `tetrad` and `pds` always apply one fixed correction regardless of
+    the data, and exist mainly as comparators.
+    """
+    import json as _json
+
+    from .inspection import edges_from_graph
+    from .utils.io import read_dataframe
+
+    outdir = _make_output_dir(ctx.obj["output_dir"], f"deconfound_{strategy}")
+
+    graph = np.load(graph_path, allow_pickle=True)
+    df = read_dataframe(data_path)
+    var_name_list = _parse_comma_list(var_names)
+    if var_name_list:
+        df.columns = var_name_list
+
+    d = df.shape[1]
+    max_lag = int(graph.shape[2] - 1)
+    # discover saves the C-node rows/columns too when include_C was set; the
+    # deconfounding layer works on observed variables only.
+    graph_obs = np.asarray(graph)[:d, :d, : max_lag + 1]
+
+    for name, value, owner in (
+        ("--threshold", threshold, "tetrad"),
+        ("--alpha", alpha, "pds"),
+    ):
+        if value is not None and strategy != owner:
+            _log(ctx, f"Note: {name} only applies to --strategy {owner}; ignoring it.")
+
+    _log(ctx, f"Graph {graph.shape} -> observed slice {graph_obs.shape}, d={d}")
+    edges_before = int(graph_obs.sum())
+
+    summary = {
+        "command": "deconfound",
+        "strategy": strategy,
+        "graph": graph_path,
+        "data": data_path,
+        "n_vars": d,
+        "max_lag": max_lag,
+        "edges_before": edges_before,
+        "output_files": {},
+    }
+
+    if strategy == "adaptive":
+        from .confounders import run_lucid
+
+        res = run_lucid(df, max_lag, discovery=graph_obs)
+        out = np.asarray(res.cg_tig)
+        summary["regime"] = res.regime
+        summary["spectral_ratio"] = res.spectral_ratio
+        summary["tau"] = res.tau
+        summary["n_factors"] = res.n_factors
+        _log(
+            ctx,
+            f"Regime: {res.regime} (R={res.spectral_ratio:.3f} vs tau={res.tau:.3f})",
+        )
+    elif strategy == "tetrad":
+        from .confounders import tetrad_filter
+
+        kw = {} if threshold is None else {"threshold": threshold}
+        out = np.asarray(tetrad_filter(df, graph_obs, max_lag, **kw))
+        summary["threshold"] = threshold if threshold is not None else 0.25
+    else:  # pds
+        from .confounders import pds_filter
+
+        # pds_filter takes alpha positionally with no default; mirror the default
+        # CausalResult.pds_filter uses so the CLI and the method agree.
+        alpha_used = 1e-10 if alpha is None else alpha
+        out = np.asarray(pds_filter(df, graph_obs, max_lag, alpha_used))
+        summary["alpha"] = alpha_used
+
+    edges_after = int(out.sum())
+    summary["edges_after"] = edges_after
+    summary["edges_removed"] = edges_before - edges_after
+
+    np.save(os.path.join(outdir, "deconfounded_graph.npy"), out)
+    summary["output_files"]["graph"] = "deconfounded_graph.npy"
+
+    names = var_name_list or [str(c) for c in df.columns]
+    summary["edges"] = edges_from_graph(out, names)
+
+    _save_json(summary, os.path.join(outdir, "summary.json"))
+    _log(
+        ctx,
+        f"Edges: {edges_before} -> {edges_after} "
+        f"({edges_before - edges_after} removed)",
+    )
+    _log(ctx, f"Results saved to {outdir}")
+
+    if output_json:
+        click.echo(_json.dumps(summary, indent=2, default=str))
+
+
 # ---------------------------------------------------------------------------
 # dowhy — effect estimation, SCM fitting, root cause analysis
 # ---------------------------------------------------------------------------

causalts/confounders/result.py

@@ -28,14 +28,19 @@ class LucidResult(CausalResult):
         The router statistic ``R`` -- the share of residual correlation mass carried by
         the top ``router_k`` eigenvalues. ``None`` for routers that do not compute it.
     tau : float or None
-        Routing threshold ``R`` was compared against. ``R > tau`` selects the pervasive
-        branch.
+        Routing threshold ``R`` was compared against. ``R <= tau`` selects ``"sparse"``;
+        ``R > tau`` selects a confounded branch -- ``"sf"`` or, above a second
+        threshold, ``"pervasive"``. So ``R > tau`` alone does **not** imply the
+        ``"pervasive"`` regime; read :attr:`regime` for the branch actually taken.
     router : str
         Which router ran (``"auto"``, ``"spectral"`` or ``"mp"``).
     n_factors : int or None
-        Number of pervasive latent factors implied by the Marchenko-Pastur edge
-        (:func:`~causalts.confounders.mp_factor_count`). ``0`` means no pervasive
-        factor was detected.
+        Number of latent factors implied by the Marchenko-Pastur edge
+        (:func:`~causalts.confounders.mp_factor_count`) -- eigenvalues too large to
+        come from the no-factor bulk. ``0`` means no such factor was detected.
+        This counts *broadly loading* factors and is independent of the
+        :attr:`regime` label: an ``"sf"`` result typically reports a nonzero
+        ``n_factors`` too.
     factor_loadings : np.ndarray or None, shape (n_factors, d)
         Leading right singular vectors of the VAR residuals, one row per detected
         factor, columns aligned with ``var_names``.

causalts/inspection.py

@@ -12,7 +12,10 @@
 Public API:
 
 - :func:`inspect_df` — full report ``{schema_version, data, facts, recommendation,
-  cost_class, warnings}`` for an in-memory DataFrame.
+  cost_class, warnings}`` for an in-memory DataFrame.  ``facts["latent_factor"]``
+  reports broad latent-factor structure when it is present; it is positive evidence
+  only, so ``detected`` False/None never means "no confounding" (see
+  :func:`_latent_factor_block`).
 - :func:`recommend_config` — the pure facts → config decision function.
 - :func:`discover_df` — run discovery on an in-memory DataFrame (the Python twin
   of ``causal-ts discover``).
@@ -198,6 +201,53 @@ def _warnings(df, data):
     return warns
 
 
+def _latent_factor_block(arr):
+    """Marchenko-Pastur check for broad latent-factor structure.
+
+    Delegates to LUCID's own router (``causalts.confounders.routed_deconf._route``)
+    rather than re-deriving the statistic, so this can never drift from the routing
+    decision :func:`~causalts.confounders.run_lucid` actually makes, and inherits its
+    validated no-factor null instead of a fresh ad-hoc cutoff.
+
+    This yields **positive evidence only**:
+
+    - ``detected=True``  — the residual spectrum carries factor structure above the
+      no-factor null, i.e. one or more latent factors load broadly enough to see.
+      This covers both regimes LUCID treats as confounded (``"sf"`` and
+      ``"pervasive"``), so do not describe it as specifically "pervasive".
+    - ``detected=False`` — no such structure. This is **not** "no confounding": a
+      latent cause touching only two or three variables produces no dominant
+      eigenvalue and is invisible to this test by construction.
+    - ``detected=None``  — the check could not run (see the guards below); this is
+      deliberately distinct from ``False`` so "did not check" is never read as
+      "checked and found nothing".
+
+    Returns
+    -------
+    dict
+        ``{detected, spectral_ratio, tau}``; the latter two are ``None`` whenever
+        the check did not run.
+    """
+    unavailable = {"detected": None, "spectral_ratio": None, "tau": None}
+    T, d = arr.shape
+    # The statistic is built on VAR(1) least-squares residuals, so skip the inputs
+    # that would make the fit raise (non-finite) or return a meaningless number
+    # (underdetermined: the design matrix is (T-1) x (d+1); d < 2 has no spectrum).
+    if d < 2 or T < d + 3 or not np.isfinite(arr).all():
+        return unavailable
+    try:
+        from .confounders.routed_deconf import _route
+
+        regime, info = _route(arr, router="auto")
+    except Exception:  # a diagnostic must never break the whole report
+        return unavailable
+    return {
+        "detected": regime != "sparse",
+        "spectral_ratio": float(info["R"]),
+        "tau": float(info["tau"]),
+    }
+
+
 def recommend_config(facts, data):
     """Map measured facts to a discovery configuration (pure, deterministic).
 
@@ -262,6 +312,18 @@ def recommend_config(facts, data):
     else:
         c_preset = "linear"
 
+    # --- latent confounding (advisory only) ---
+    # Deliberately does NOT influence algorithm/ci_test/include_C: LUCID is orthogonal
+    # post-discovery correction, not an alternative discovery algorithm. And the nudge
+    # fires only on a positive detection -- `detected` False/None cannot rule out
+    # sparse (few-variable) confounding, so silence is the honest default.
+    lf = facts.get("latent_factor") or {}
+    if lf.get("detected") is True:
+        reasons.append(
+            f"latent factor detected (R={lf['spectral_ratio']:.2f} vs "
+            f"tau={lf['tau']:.2f}) → consider res.deconfound() after discovery"
+        )
+
     return {
         "algorithm": algorithm,
         "ci_test": ci_test,
@@ -324,6 +386,7 @@ def inspect_df(df, max_lag=None):
             "form": form,
         },
         "suggested_max_lag": int(suggested),
+        "latent_factor": _latent_factor_block(arr),
     }
 
     recommendation = recommend_config(facts, data)

causalts/skills/causal-ts-discovery/SKILL.md

@@ -46,7 +46,14 @@ warnings}`. Read it — do **not** re-derive these facts yourself.
   too-few rows). If a column is heavily missing or constant, recommend fixing it
   (impute/drop) before trusting results.
 - **`facts`** — linearity, per-column (non)stationarity + its `form`, suggested
-  max lag.
+  max lag, and `latent_factor`.
+  - **`latent_factor`** — `{detected, spectral_ratio, tau}`. `detected: true` means
+    one or more latent factors load broadly across the panel; plan on deconfounding
+    (§5). Read the negatives narrowly: `false` only rules out factor structure this
+    test can see — a latent cause touching just two or three variables produces no
+    dominant eigenvalue and stays invisible — and `null` means the check could not run
+    (missing values, `d < 2`, too few rows). **Never report either as "no
+    confounding".**
 - **`recommendation`** — `{algorithm, ci_test, include_C, c_preset, max_lag,
   rationale}`. This is a deterministic default; you may **override it** with
   context the tool can't see (e.g. the user says "these are already
@@ -116,6 +123,15 @@ read it rather than eyeballing, then apply:
   differencing, or verifying the C-node preset matches the trend `form`.
 - **High `max_in_degree` at `hub`** → inspect whether that variable is a common
   effect or an artifact of a confounder/persistence.
+- **Many `contemporaneous` edges, especially with step-2 `latent_factor.detected`**
+  → a broadly-loading latent factor induces exactly this signature (lag-0 edges
+  among variables with no direct causal link). Offer to correct it:
+  `causal-ts deconfound <graph.npy> --data <data-file>` (or `res.deconfound()` in
+  Python), which infers whether the confounding is sparse or pervasive and applies
+  the matching correction. Report edges-before/after and be explicit that the
+  removed edges were **judged confounded, not disproven**. Which edges are eligible
+  depends on the inferred regime: the pervasive branch rewrites the lag-0 slice only,
+  while the sparse branch also tests lagged edges against observed controls.
 - **`lagged` == 0 (all edges contemporaneous)** → check that `max_lag` is
   adequate and the sampling rate isn't washing out dynamics.
 - **Self-loops** are autoregressive terms (a variable's own past), expected for