diff --git a/.github/workflows/sym-arch-002a7-multiplicity.yml b/.github/workflows/sym-arch-002a7-multiplicity.yml new file mode 100644 index 000000000..3ba4d5cba --- /dev/null +++ b/.github/workflows/sym-arch-002a7-multiplicity.yml @@ -0,0 +1,68 @@ +name: SYM-ARCH-002A7 Multiplicity + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: + - main + - research/sym-arch-002a2-hierarchical-power-v1 + paths: + - '.github/workflows/sym-arch-002a7-multiplicity.yml' + - 'crates/domains/symthaea-psych-bench/src/experiment/multiplicity.rs' + - 'crates/domains/symthaea-psych-bench/src/lib.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' + - 'docs/research/SYM_ARCH_002A7_MULTIPLICITY_V1.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +concurrency: + # Automatic branch/PR validation keeps only the newest run. Manual validation + # is deliberate and receives a unique group through github.run_id. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + multiplicity: + name: Validate multiplicity-safe analysis plan + # Draft stacked PRs cannot merge and should not consume scarce runner capacity. + # Promoting to ready-for-review explicitly triggers this exact gate. + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-002a7-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-002a7- + + - name: Check formatting + run: | + cargo fmt --all -- --check + + - name: Run A7 multiplicity tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_multiplicity:: -- --nocapture + + - name: Check psych-bench library + run: | + cargo check -p symthaea-psych-bench --lib diff --git a/crates/domains/symthaea-psych-bench/src/experiment/multiplicity.rs b/crates/domains/symthaea-psych-bench/src/experiment/multiplicity.rs new file mode 100644 index 000000000..15ab36e65 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/multiplicity.rs @@ -0,0 +1,434 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Multiplicity-safe analysis-plan primitives for SYM-ARCH-002A7. +//! +//! This module does not manufacture p-values or effect estimates. It freezes the +//! claim-bearing hypothesis family and applies standard family-wise corrections to +//! valid preregistered inferential outputs. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +pub const MULTIPLICITY_PLAN_SCHEMA_V1: &str = "symthaea.multiplicity-plan/v1"; +const MULTIPLICITY_PLAN_HASH_DOMAIN: &[u8] = b"symthaea.multiplicity-plan.hash/v1"; + +fn canonical_hash(domain: &[u8], value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?; + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(&[0]); + hasher.update(&bytes); + Ok(hasher.finalize().to_hex().to_string()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HypothesisTail { + Greater, + Less, + TwoSided, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HypothesisRole { + Primary, + Secondary, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HypothesisSpec { + pub hypothesis_id: String, + pub tail: HypothesisTail, + pub role: HypothesisRole, +} + +impl HypothesisSpec { + fn validate(&self) -> Result<(), String> { + if self.hypothesis_id.trim().is_empty() || self.hypothesis_id.trim() != self.hypothesis_id { + return Err("hypothesis ids must be non-empty and already normalized".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MultiplicityPlan { + pub schema: String, + pub family_id: String, + /// Family-wise Type-I error rate. + pub family_alpha: f64, + pub hypotheses: Vec, +} + +impl MultiplicityPlan { + pub fn validate(&self) -> Result<(), String> { + if self.schema != MULTIPLICITY_PLAN_SCHEMA_V1 { + return Err(format!("unsupported multiplicity-plan schema: {}", self.schema)); + } + if self.family_id.trim().is_empty() || self.family_id.trim() != self.family_id { + return Err("family_id must be non-empty and already normalized".into()); + } + if !self.family_alpha.is_finite() + || self.family_alpha <= 0.0 + || self.family_alpha >= 1.0 + { + return Err("family_alpha must be finite in (0,1)".into()); + } + if self.hypotheses.is_empty() { + return Err("multiplicity family must contain at least one hypothesis".into()); + } + let mut ids = BTreeSet::new(); + for hypothesis in &self.hypotheses { + hypothesis.validate()?; + if !ids.insert(hypothesis.hypothesis_id.as_str()) { + return Err("duplicate hypothesis id in multiplicity family".into()); + } + } + Ok(()) + } + + /// Canonical digest independent of the input ordering of hypothesis specs. + pub fn digest(&self) -> Result { + self.validate()?; + let mut hypotheses = self.hypotheses.clone(); + hypotheses.sort_by(|left, right| left.hypothesis_id.cmp(&right.hypothesis_id)); + canonical_hash( + MULTIPLICITY_PLAN_HASH_DOMAIN, + &( + self.schema.as_str(), + self.family_id.as_str(), + self.family_alpha, + hypotheses, + ), + ) + } + + pub fn hypothesis_count(&self) -> usize { + self.hypotheses.len() + } + + /// Per-comparison alpha for Bonferroni simultaneous confidence intervals. + pub fn bonferroni_per_comparison_alpha(&self) -> Result { + self.validate()?; + Ok(self.family_alpha / self.hypothesis_count() as f64) + } + + pub fn bonferroni_confidence_level(&self) -> Result { + Ok(1.0 - self.bonferroni_per_comparison_alpha()?) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RawHypothesisPValue { + pub hypothesis_id: String, + /// Tail used by the separately frozen raw inferential test. + pub tail: HypothesisTail, + /// Raw p-value produced by that separately frozen test. + pub raw_p: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct HolmAdjustedHypothesis { + pub hypothesis_id: String, + pub tail: HypothesisTail, + pub role: HypothesisRole, + pub raw_p: f64, + pub holm_adjusted_p: f64, + pub reject_at_family_alpha: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct HolmFamilyResult { + pub family_id: String, + pub plan_digest: String, + pub family_alpha: f64, + pub hypotheses: Vec, +} + +/// Apply Holm's step-down family-wise correction. +/// +/// Raw p-values must correspond exactly to the hypothesis ids and tails frozen in +/// the plan. This function cannot verify how a p-value was generated; that test +/// remains part of the preregistered analysis contract. +pub fn apply_holm( + plan: &MultiplicityPlan, + raw: &[RawHypothesisPValue], +) -> Result { + plan.validate()?; + if raw.len() != plan.hypothesis_count() { + return Err("raw p-value count must match the frozen hypothesis family".into()); + } + + let mut raw_ids = BTreeSet::new(); + for value in raw { + if value.hypothesis_id.trim().is_empty() || value.hypothesis_id.trim() != value.hypothesis_id { + return Err("raw p-value hypothesis ids must be non-empty and normalized".into()); + } + if !value.raw_p.is_finite() || !(0.0..=1.0).contains(&value.raw_p) { + return Err("raw p-values must be finite in [0,1]".into()); + } + if !raw_ids.insert(value.hypothesis_id.as_str()) { + return Err("duplicate hypothesis id in raw p-values".into()); + } + } + + let planned_ids: BTreeSet<&str> = plan + .hypotheses + .iter() + .map(|hypothesis| hypothesis.hypothesis_id.as_str()) + .collect(); + if raw_ids != planned_ids { + return Err("raw p-value hypothesis ids do not match frozen family".into()); + } + + let mut work: Vec<(HypothesisSpec, f64)> = Vec::with_capacity(plan.hypothesis_count()); + for spec in &plan.hypotheses { + let observed = raw + .iter() + .find(|value| value.hypothesis_id == spec.hypothesis_id) + .expect("id-set equality established"); + if observed.tail != spec.tail { + return Err(format!( + "raw p-value tail for {} does not match frozen hypothesis tail", + spec.hypothesis_id + )); + } + work.push((spec.clone(), observed.raw_p)); + } + + // Deterministic tie-break by frozen hypothesis id. + work.sort_by(|left, right| { + left.1 + .total_cmp(&right.1) + .then_with(|| left.0.hypothesis_id.cmp(&right.0.hypothesis_id)) + }); + + let count = work.len(); + let mut running_adjusted = 0.0f64; + let mut adjusted = Vec::with_capacity(count); + for (rank, (spec, raw_p)) in work.into_iter().enumerate() { + let step_adjusted = ((count - rank) as f64 * raw_p).min(1.0); + running_adjusted = running_adjusted.max(step_adjusted); + adjusted.push(HolmAdjustedHypothesis { + hypothesis_id: spec.hypothesis_id, + tail: spec.tail, + role: spec.role, + raw_p, + holm_adjusted_p: running_adjusted, + reject_at_family_alpha: running_adjusted <= plan.family_alpha, + }); + } + + // Output ordering is canonical by id, not by observed significance rank. + adjusted.sort_by(|left, right| left.hypothesis_id.cmp(&right.hypothesis_id)); + Ok(HolmFamilyResult { + family_id: plan.family_id.clone(), + plan_digest: plan.digest()?, + family_alpha: plan.family_alpha, + hypotheses: adjusted, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn plan(order: &[&str]) -> MultiplicityPlan { + MultiplicityPlan { + schema: MULTIPLICITY_PLAN_SCHEMA_V1.into(), + family_id: "sym-arch-002-primary-family".into(), + family_alpha: 0.05, + hypotheses: order + .iter() + .enumerate() + .map(|(index, id)| HypothesisSpec { + hypothesis_id: (*id).into(), + tail: if index % 2 == 0 { + HypothesisTail::Greater + } else { + HypothesisTail::TwoSided + }, + role: HypothesisRole::Primary, + }) + .collect(), + } + } + + #[test] + fn holm_matches_known_step_down_example() { + let plan = MultiplicityPlan { + schema: MULTIPLICITY_PLAN_SCHEMA_V1.into(), + family_id: "family".into(), + family_alpha: 0.05, + hypotheses: vec![ + HypothesisSpec { + hypothesis_id: "h1".into(), + tail: HypothesisTail::Greater, + role: HypothesisRole::Primary, + }, + HypothesisSpec { + hypothesis_id: "h2".into(), + tail: HypothesisTail::Greater, + role: HypothesisRole::Primary, + }, + HypothesisSpec { + hypothesis_id: "h3".into(), + tail: HypothesisTail::Greater, + role: HypothesisRole::Primary, + }, + ], + }; + let result = apply_holm( + &plan, + &[ + RawHypothesisPValue { + hypothesis_id: "h1".into(), + tail: HypothesisTail::Greater, + raw_p: 0.01, + }, + RawHypothesisPValue { + hypothesis_id: "h2".into(), + tail: HypothesisTail::Greater, + raw_p: 0.04, + }, + RawHypothesisPValue { + hypothesis_id: "h3".into(), + tail: HypothesisTail::Greater, + raw_p: 0.03, + }, + ], + ) + .unwrap(); + let h1 = result + .hypotheses + .iter() + .find(|hypothesis| hypothesis.hypothesis_id == "h1") + .unwrap(); + let h2 = result + .hypotheses + .iter() + .find(|hypothesis| hypothesis.hypothesis_id == "h2") + .unwrap(); + let h3 = result + .hypotheses + .iter() + .find(|hypothesis| hypothesis.hypothesis_id == "h3") + .unwrap(); + assert!((h1.holm_adjusted_p - 0.03).abs() < 1e-12); + assert!((h2.holm_adjusted_p - 0.06).abs() < 1e-12); + assert!((h3.holm_adjusted_p - 0.06).abs() < 1e-12); + assert!(h1.reject_at_family_alpha); + assert!(!h2.reject_at_family_alpha); + assert!(!h3.reject_at_family_alpha); + } + + #[test] + fn plan_digest_is_order_independent_when_specs_are_identical() { + let first = MultiplicityPlan { + schema: MULTIPLICITY_PLAN_SCHEMA_V1.into(), + family_id: "family".into(), + family_alpha: 0.05, + hypotheses: vec![ + HypothesisSpec { + hypothesis_id: "a".into(), + tail: HypothesisTail::Greater, + role: HypothesisRole::Primary, + }, + HypothesisSpec { + hypothesis_id: "b".into(), + tail: HypothesisTail::TwoSided, + role: HypothesisRole::Secondary, + }, + ], + }; + let second = MultiplicityPlan { + hypotheses: first.hypotheses.iter().cloned().rev().collect(), + ..first.clone() + }; + assert_eq!(first.digest().unwrap(), second.digest().unwrap()); + } + + #[test] + fn plan_digest_changes_when_tail_changes() { + let first = plan(&["a", "b"]); + let mut second = first.clone(); + second.hypotheses[0].tail = HypothesisTail::Less; + assert_ne!(first.digest().unwrap(), second.digest().unwrap()); + } + + #[test] + fn bonferroni_interval_alpha_controls_family_rate() { + let plan = MultiplicityPlan { + schema: MULTIPLICITY_PLAN_SCHEMA_V1.into(), + family_id: "family".into(), + family_alpha: 0.05, + hypotheses: (0..4) + .map(|index| HypothesisSpec { + hypothesis_id: format!("h{index}"), + tail: HypothesisTail::TwoSided, + role: HypothesisRole::Secondary, + }) + .collect(), + }; + assert!((plan.bonferroni_per_comparison_alpha().unwrap() - 0.0125).abs() < 1e-12); + assert!((plan.bonferroni_confidence_level().unwrap() - 0.9875).abs() < 1e-12); + } + + #[test] + fn raw_family_must_match_frozen_ids_exactly() { + let plan = plan(&["a", "b"]); + let missing = [RawHypothesisPValue { + hypothesis_id: "a".into(), + tail: HypothesisTail::Greater, + raw_p: 0.01, + }]; + assert!(apply_holm(&plan, &missing).is_err()); + + let wrong = [ + RawHypothesisPValue { + hypothesis_id: "a".into(), + tail: HypothesisTail::Greater, + raw_p: 0.01, + }, + RawHypothesisPValue { + hypothesis_id: "c".into(), + tail: HypothesisTail::TwoSided, + raw_p: 0.02, + }, + ]; + assert!(apply_holm(&plan, &wrong).is_err()); + } + + #[test] + fn raw_tail_must_match_frozen_tail() { + let plan = plan(&["a"]); + let raw = [RawHypothesisPValue { + hypothesis_id: "a".into(), + tail: HypothesisTail::Less, + raw_p: 0.01, + }]; + assert!(apply_holm(&plan, &raw).is_err()); + } + + #[test] + fn non_finite_or_out_of_range_p_values_fail_closed() { + let plan = plan(&["a"]); + for raw_p in [f64::NAN, -0.01, 1.01] { + let raw = [RawHypothesisPValue { + hypothesis_id: "a".into(), + tail: HypothesisTail::Greater, + raw_p, + }]; + assert!(apply_holm(&plan, &raw).is_err()); + } + } + + #[test] + fn duplicate_hypothesis_ids_fail_closed() { + let mut invalid = plan(&["a", "b"]); + invalid.hypotheses[1].hypothesis_id = "a".into(); + assert!(invalid.validate().is_err()); + } +} diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index 0e12062ca..06057e331 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -56,6 +56,8 @@ pub mod benchmarks; pub mod experiment; #[path = "experiment/confirmatory.rs"] pub mod experiment_confirmatory; +#[path = "experiment/multiplicity.rs"] +pub mod experiment_multiplicity; #[path = "experiment/statistics.rs"] pub mod experiment_statistics; #[path = "experiment/statistics_design.rs"] @@ -68,4 +70,4 @@ pub mod wm; pub mod neuroevolution_fitness; #[cfg(test)] -mod proptest_math_benchmarks; \ No newline at end of file +mod proptest_math_benchmarks; diff --git a/docs/research/SYM_ARCH_002A7_MULTIPLICITY_V1.md b/docs/research/SYM_ARCH_002A7_MULTIPLICITY_V1.md new file mode 100644 index 000000000..7ec7f93fd --- /dev/null +++ b/docs/research/SYM_ARCH_002A7_MULTIPLICITY_V1.md @@ -0,0 +1,151 @@ +# SYM-ARCH-002A7 — Multiplicity-Safe Confirmatory Families v1 + +**Status:** statistical analysis-plan infrastructure; no architecture result + +**Tracks:** #55 + +**Base:** SYM-ARCH-002A2 hierarchical statistics and power (#58) + +## Why this exists + +SYM-ARCH-002 intentionally measures several phenomena and may compare more than one architecture/control. Without a frozen hypothesis family and multiplicity correction, a later analysis could accidentally select the most favorable comparison or metric and report an ordinary nominal p-value as though it were the only test performed. + +A7 makes the claim-bearing hypothesis family explicit before outcomes are interpreted. + +It does **not** generate effect estimates or raw p-values. It only freezes family membership/tails and applies family-wise correction to inferential outputs produced by separately preregistered tests. + +## Frozen hypothesis family + +`MultiplicityPlan` records: + +- schema/version; +- normalized `family_id`; +- family-wise alpha; +- exact hypothesis ids; +- frozen tail for every hypothesis (`greater`, `less`, or `two_sided`); +- primary/secondary role for every hypothesis. + +Hypothesis ids must be unique. The plan receives a domain-separated BLAKE3 digest that is independent of input ordering but changes when family membership, role, tail, alpha, or family identity changes. + +A claim-bearing analysis must bind the exact plan digest before observing CONFIRM outcomes. + +## Raw inferential inputs + +`RawHypothesisPValue` records: + +- frozen hypothesis id; +- tail actually used by the raw test; +- raw p-value. + +A7 fails closed unless: + +- the raw family contains exactly the frozen hypothesis ids; +- no hypothesis appears more than once; +- every p-value is finite and in `[0,1]`; +- every submitted tail exactly matches the tail frozen in the plan. + +This prevents a one-sided/two-sided or direction switch from being hidden inside the multiplicity layer. + +A7 cannot verify that the upstream raw test itself was implemented correctly. Test statistic, resampling method, pairing unit, nuisance topology, and seed policy remain part of the separately frozen A2/experiment analysis contract. + +## Holm family-wise correction + +`apply_holm` implements Holm's step-down family-wise correction: + +1. sort raw p-values ascending; +2. deterministic tie-break by frozen hypothesis id; +3. multiply each ordered p-value by the number of hypotheses remaining; +4. cap at `1.0`; +5. take the running maximum so adjusted p-values are monotone; +6. compare adjusted p-values to the frozen family alpha. + +The returned result is canonicalized by hypothesis id rather than significance rank. + +This follows the same correction pattern already used by Symthaea's Muse confirmatory-study analysis, but A7 is implemented independently in psych-bench so architecture research is not coupled to Muse-specific endpoint types. + +## Simultaneous confidence-interval support + +A7 also exposes: + +- Bonferroni per-comparison alpha = `family_alpha / m`; +- corresponding confidence level = `1 - family_alpha/m`. + +This permits a separately implemented interval estimator to request conservative simultaneous confidence intervals for a frozen family. + +Holm-adjusted p-values and Bonferroni simultaneous intervals are **two reporting tools**, not ingredients of a single blended score. + +A later analysis must preregister which inferential path supports each claim. It must not choose between nominal, Holm, or Bonferroni reporting after seeing which yields the preferred conclusion. + +## Relationship to SESOI / practical effects + +Multiplicity control and practical-effect control answer different questions: + +- multiplicity: how much false-positive risk is created by a family of tests? +- SESOI: is the estimated effect large enough to matter? + +A statistically significant but practically tiny effect does not pass the architecture claim gate merely because Holm rejects it. + +Likewise, a large point estimate does not become confirmatory evidence if its multiplicity-safe inferential gate fails. + +Claim-bearing architecture results should therefore preserve both dimensions separately in the ClaimLedger/evidence record. + +## Primary vs secondary hypotheses + +`HypothesisRole` records whether a hypothesis was frozen as primary or secondary. + +The role is part of the plan digest. A hypothesis cannot be relabeled primary after outcomes are observed without producing a different plan identity and therefore a new/post-hoc analysis specification. + +A7 does not impose one universal rule about whether primary and secondary hypotheses belong in the same family. The preregistration must define the family structure in advance and justify it. Splitting one scientific family into several smaller families after observing results is not permitted. + +## What must be frozen before CONFIRM + +At minimum: + +- exact family id(s); +- exact hypothesis ids; +- hypothesis role; +- direction/tail; +- family alpha; +- upstream raw test for each hypothesis; +- pairing/generalization unit; +- nuisance topology; +- metric and comparator; +- SESOI/practical-effect rule; +- multiplicity method; +- interval method/alpha if simultaneous intervals are used; +- code/analysis revision and plan digest. + +DEV may be used to choose this structure. CONFIRM/REPL may not change it after behavioral outcomes are observed. + +## Acceptance tests + +The exact PR head must demonstrate: + +1. known Holm step-down example produces expected adjusted p-values; +2. plan digest is independent of hypothesis serialization order; +3. changing a frozen tail changes plan identity; +4. Bonferroni family alpha is computed correctly; +5. raw hypothesis ids must match the frozen family exactly; +6. raw test tail must match the frozen tail; +7. duplicate hypothesis ids fail closed; +8. NaN/out-of-range p-values fail closed; +9. deterministic tie handling is stable; +10. psych-bench library compiles. + +## Claim ceiling + +Merging A7 supports only: + +> Symthaea psych-bench can freeze claim-bearing hypothesis families and apply deterministic family-wise multiplicity control to separately preregistered inferential outputs. + +It does **not** support: + +- a Symthaea performance claim; +- a claim that any raw p-value is valid merely because A7 accepts its numeric range; +- a practical-effect claim without SESOI/effect-size evidence; +- post-hoc family splitting or tail switching; +- a claim that Holm and Bonferroni results should be averaged or collapsed into one score. + +## Next use + +After A2/A7 and the rest of the A-series are executable and green, freeze the exact primary hypothesis family for a DEV dry run, validate the complete analysis path, and only then commit the untouched CONFIRM family/plan digest.