From 941379511dc19a7b27b750e9549c010b0efdd5da Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:01:43 +0200 Subject: [PATCH 01/13] feat(research): add hierarchical inference and prospective power --- .../src/experiment/statistics.rs | 412 ++++++++++++++++++ 1 file changed, 412 insertions(+) create mode 100644 crates/domains/symthaea-psych-bench/src/experiment/statistics.rs diff --git a/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs new file mode 100644 index 000000000..8665e3cd7 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs @@ -0,0 +1,412 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Hierarchical uncertainty and prospective power planning for architecture research. +//! +//! Confirmatory inference treats independently generated environments as the +//! scientific unit of generalization. Nested representation/learner/stream runs +//! quantify within-environment uncertainty, but do not increase the number of +//! independent environments. + +use crate::experiment::confirmatory::{PracticalEffect, classify_practical_effect}; +use crate::experiment::PairedEstimate; +use crate::harness::analysis::bootstrap_ci_bca; +use rand::rngs::StdRng; +use rand::{Rng, RngCore, SeedableRng}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +fn looks_like_digest(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) +} + +fn percentile(sorted: &[f64], probability: f64) -> f64 { + debug_assert!(!sorted.is_empty()); + let position = probability.clamp(0.0, 1.0) * (sorted.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + sorted[lower] + } else { + let weight = position - lower as f64; + sorted[lower] * (1.0 - weight) + sorted[upper] * weight + } +} + +/// Candidate/control outcomes nested within one independently generated world. +/// +/// Each run index must represent the same nuisance realization for candidate and +/// control (for example the same representation/learner/stream seed tuple). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NestedEnvironmentResult { + pub environment_digest: String, + pub candidate_runs: Vec, + pub control_runs: Vec, +} + +impl NestedEnvironmentResult { + pub fn validate(&self) -> Result<(), String> { + if !looks_like_digest(&self.environment_digest) { + return Err("environment digest must be a 32-byte hex digest".into()); + } + if self.candidate_runs.is_empty() || self.candidate_runs.len() != self.control_runs.len() { + return Err("candidate/control nested runs must be non-empty and paired".into()); + } + if self + .candidate_runs + .iter() + .chain(&self.control_runs) + .any(|value| !value.is_finite()) + { + return Err("nested run outcomes must be finite".into()); + } + Ok(()) + } + + pub fn run_count(&self) -> usize { + self.candidate_runs.len() + } + + pub fn mean_delta(&self) -> f64 { + self.candidate_runs + .iter() + .zip(&self.control_runs) + .map(|(candidate, control)| candidate - control) + .sum::() + / self.run_count() as f64 + } +} + +fn validate_environment_results(results: &[NestedEnvironmentResult]) -> Result<(), String> { + if results.len() < 3 { + return Err("at least three independent environments are required".into()); + } + let mut seen = BTreeSet::new(); + for result in results { + result.validate()?; + if !seen.insert(result.environment_digest.to_ascii_lowercase()) { + return Err("duplicate environment digest would create pseudoreplication".into()); + } + } + Ok(()) +} + +/// Equal-environment-weight hierarchical bootstrap estimate. +/// +/// The bootstrap first samples environments with replacement. For each sampled +/// environment it then samples paired nested runs with replacement and computes +/// one environment mean delta. The final replicate is the equal-weight mean of +/// sampled environment means, so environments with more nuisance runs do not +/// receive more scientific weight. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct HierarchicalEstimate { + pub n_environments: usize, + pub total_nested_pairs: usize, + pub mean_delta: f64, + pub ci95_low: f64, + pub ci95_high: f64, + pub bootstrap_resamples: usize, +} + +pub fn hierarchical_environment_delta_percentile( + results: &[NestedEnvironmentResult], + n_resamples: usize, + seed: u64, +) -> Result { + validate_environment_results(results)?; + if n_resamples < 200 { + return Err("hierarchical bootstrap requires at least 200 resamples".into()); + } + + let n_environments = results.len(); + let total_nested_pairs = results.iter().map(NestedEnvironmentResult::run_count).sum(); + let mean_delta = results + .iter() + .map(NestedEnvironmentResult::mean_delta) + .sum::() + / n_environments as f64; + + let mut rng = StdRng::seed_from_u64(seed); + let mut bootstrap = Vec::with_capacity(n_resamples); + + for _ in 0..n_resamples { + let mut environment_sum = 0.0; + for _ in 0..n_environments { + let environment = &results[rng.gen_range(0..n_environments)]; + let mut nested_sum = 0.0; + for _ in 0..environment.run_count() { + let run = rng.gen_range(0..environment.run_count()); + nested_sum += environment.candidate_runs[run] - environment.control_runs[run]; + } + environment_sum += nested_sum / environment.run_count() as f64; + } + bootstrap.push(environment_sum / n_environments as f64); + } + + bootstrap.sort_by(|left, right| left.total_cmp(right)); + Ok(HierarchicalEstimate { + n_environments, + total_nested_pairs, + mean_delta, + ci95_low: percentile(&bootstrap, 0.025), + ci95_high: percentile(&bootstrap, 0.975), + bootstrap_resamples: n_resamples, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PowerDirection { + MeaningfulGain, + MeaningfulRegression, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProspectivePowerConfig { + /// Candidate numbers of independent environments, evaluated in ascending order. + pub environment_counts: Vec, + /// Number of nested paired runs planned per future environment. + pub runs_per_environment: usize, + /// Monte Carlo future-study simulations per candidate environment count. + pub simulation_trials: usize, + /// BCa resamples used inside each simulated future study. + pub bootstrap_resamples: usize, + /// Desired probability of clearing the practical-effect gate. + pub target_power: f64, + /// Frozen smallest effect size of interest in the primary metric's natural units. + pub sesoi: f64, + pub direction: PowerDirection, + pub seed: u64, +} + +impl ProspectivePowerConfig { + pub fn validate(&self) -> Result<(), String> { + if self.environment_counts.is_empty() { + return Err("at least one candidate environment count is required".into()); + } + if self.environment_counts.iter().any(|count| *count < 3) { + return Err("candidate environment counts must be at least three".into()); + } + if self.environment_counts.windows(2).any(|window| window[0] >= window[1]) { + return Err("candidate environment counts must be strictly increasing".into()); + } + if self.runs_per_environment == 0 { + return Err("runs_per_environment must be positive".into()); + } + if self.simulation_trials < 100 { + return Err("prospective power requires at least 100 simulation trials".into()); + } + if self.bootstrap_resamples < 100 { + return Err("prospective power requires at least 100 bootstrap resamples".into()); + } + if !self.target_power.is_finite() || !(0.5..=1.0).contains(&self.target_power) { + return Err("target power must be finite and between 0.5 and 1.0".into()); + } + if !self.sesoi.is_finite() || self.sesoi <= 0.0 { + return Err("SESOI must be finite and strictly positive".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PowerPoint { + pub environments: usize, + pub successes: usize, + pub simulation_trials: usize, + pub estimated_power: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProspectivePowerPlan { + pub target_power: f64, + pub sesoi: f64, + pub direction: PowerDirection, + pub runs_per_environment: usize, + pub minimum_environments: Option, + pub points: Vec, +} + +/// Estimate a future CONFIRM sample size using DEV outcomes only. +/// +/// This is an empirical Monte Carlo planning tool, not an analytic guarantee. +/// Each future environment is sampled from the DEV environment distribution, +/// then nested paired runs are resampled within that environment. The simulated +/// study is counted as successful only when its environment-level BCa interval +/// satisfies the same SESOI gate used for practical-effect interpretation. +/// +/// Freeze the chosen sample size before observing CONFIRM outcomes. +pub fn prospective_power_from_dev( + dev_results: &[NestedEnvironmentResult], + config: &ProspectivePowerConfig, +) -> Result { + validate_environment_results(dev_results)?; + config.validate()?; + + let mut rng = StdRng::seed_from_u64(config.seed); + let mut points = Vec::with_capacity(config.environment_counts.len()); + let mut minimum_environments = None; + + for &environment_count in &config.environment_counts { + let mut successes = 0usize; + + for _ in 0..config.simulation_trials { + let mut future_environment_means = Vec::with_capacity(environment_count); + for _ in 0..environment_count { + let template = &dev_results[rng.gen_range(0..dev_results.len())]; + let mut delta_sum = 0.0; + for _ in 0..config.runs_per_environment { + let run = rng.gen_range(0..template.run_count()); + delta_sum += template.candidate_runs[run] - template.control_runs[run]; + } + future_environment_means.push(delta_sum / config.runs_per_environment as f64); + } + + let mean_delta = future_environment_means.iter().sum::() + / future_environment_means.len() as f64; + let ci_seed = rng.next_u64(); + let (ci95_low, ci95_high) = bootstrap_ci_bca( + &future_environment_means, + config.bootstrap_resamples, + 0.05, + ci_seed, + ); + let estimate = PairedEstimate { + n_pairs: future_environment_means.len(), + mean_delta, + ci95_low, + ci95_high, + }; + let practical = classify_practical_effect(&estimate, config.sesoi)?; + let success = matches!( + (config.direction, practical), + (PowerDirection::MeaningfulGain, PracticalEffect::MeaningfulGain) + | ( + PowerDirection::MeaningfulRegression, + PracticalEffect::MeaningfulRegression + ) + ); + if success { + successes += 1; + } + } + + let estimated_power = successes as f64 / config.simulation_trials as f64; + if minimum_environments.is_none() && estimated_power >= config.target_power { + minimum_environments = Some(environment_count); + } + points.push(PowerPoint { + environments: environment_count, + successes, + simulation_trials: config.simulation_trials, + estimated_power, + }); + } + + Ok(ProspectivePowerPlan { + target_power: config.target_power, + sesoi: config.sesoi, + direction: config.direction, + runs_per_environment: config.runs_per_environment, + minimum_environments, + points, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digest(value: u64) -> String { + format!("{value:064x}") + } + + fn environment(id: u64, deltas: &[f64]) -> NestedEnvironmentResult { + NestedEnvironmentResult { + environment_digest: digest(id), + candidate_runs: deltas.iter().map(|delta| 0.5 + delta).collect(), + control_runs: vec![0.5; deltas.len()], + } + } + + #[test] + fn hierarchical_estimate_equal_weights_environments_not_run_counts() { + let results = vec![ + environment(1, &[0.20]), + environment(2, &[0.00; 10]), + environment(3, &[0.10, 0.10]), + ]; + let estimate = hierarchical_environment_delta_percentile(&results, 500, 42).unwrap(); + assert_eq!(estimate.n_environments, 3); + assert_eq!(estimate.total_nested_pairs, 13); + assert!((estimate.mean_delta - 0.10).abs() < 1e-12); + assert!(estimate.ci95_low <= estimate.mean_delta); + assert!(estimate.ci95_high >= estimate.mean_delta); + } + + #[test] + fn hierarchical_estimate_is_deterministic_for_fixed_seed() { + let results = vec![ + environment(1, &[0.08, 0.10, 0.12]), + environment(2, &[0.06, 0.09, 0.11]), + environment(3, &[0.07, 0.10, 0.13]), + environment(4, &[0.09, 0.11, 0.14]), + ]; + let first = hierarchical_environment_delta_percentile(&results, 500, 7).unwrap(); + let second = hierarchical_environment_delta_percentile(&results, 500, 7).unwrap(); + assert_eq!(first, second); + } + + #[test] + fn hierarchical_estimate_rejects_duplicate_environment_identity() { + let results = vec![ + environment(1, &[0.1]), + environment(1, &[0.2]), + environment(2, &[0.3]), + ]; + assert!(hierarchical_environment_delta_percentile(&results, 500, 42).is_err()); + } + + #[test] + fn prospective_power_plan_is_deterministic_and_finds_strong_effect() { + let dev = vec![ + environment(1, &[0.11, 0.12, 0.13]), + environment(2, &[0.10, 0.12, 0.14]), + environment(3, &[0.12, 0.13, 0.15]), + environment(4, &[0.09, 0.11, 0.13]), + environment(5, &[0.11, 0.13, 0.14]), + environment(6, &[0.10, 0.12, 0.13]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3, 5, 8], + runs_per_environment: 3, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + direction: PowerDirection::MeaningfulGain, + seed: 99, + }; + let first = prospective_power_from_dev(&dev, &config).unwrap(); + let second = prospective_power_from_dev(&dev, &config).unwrap(); + assert_eq!(first, second); + assert_eq!(first.points.len(), 3); + assert!(first.minimum_environments.is_some()); + assert!(first.points.iter().all(|point| (0.0..=1.0).contains(&point.estimated_power))); + } + + #[test] + fn prospective_power_rejects_unsorted_environment_counts() { + let config = ProspectivePowerConfig { + environment_counts: vec![8, 5], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + assert!(config.validate().is_err()); + } +} From 43d7803b758cb895e7995c26a0a60f046edb4625 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:04:20 +0200 Subject: [PATCH 02/13] refactor(research): harden hierarchical power planning --- .../src/experiment/statistics.rs | 249 ++++++++++++++---- 1 file changed, 200 insertions(+), 49 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs index 8665e3cd7..6071056e3 100644 --- a/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs +++ b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs @@ -8,14 +8,16 @@ //! quantify within-environment uncertainty, but do not increase the number of //! independent environments. -use crate::experiment::confirmatory::{PracticalEffect, classify_practical_effect}; -use crate::experiment::PairedEstimate; +use crate::experiment::{ExperimentManifest, PairedEstimate, StreamNamespace, TuningStatus}; +use crate::experiment_confirmatory::{PracticalEffect, classify_practical_effect}; use crate::harness::analysis::bootstrap_ci_bca; use rand::rngs::StdRng; use rand::{Rng, RngCore, SeedableRng}; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; +const POWER_PLAN_INPUT_DOMAIN: &[u8] = b"symthaea.prospective-power.input/v1"; + fn looks_like_digest(value: &str) -> bool { value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) } @@ -33,15 +35,61 @@ fn percentile(sorted: &[f64], probability: f64) -> f64 { } } -/// Candidate/control outcomes nested within one independently generated world. +fn digest_serialized(value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?; + let mut hasher = blake3::Hasher::new(); + hasher.update(POWER_PLAN_INPUT_DOMAIN); + hasher.update(&[0]); + hasher.update(&bytes); + Ok(hasher.finalize().to_hex().to_string()) +} + +fn wilson_interval(successes: usize, trials: usize) -> (f64, f64) { + debug_assert!(trials > 0); + let z = 1.959_963_984_540_054_f64; + let n = trials as f64; + let p = successes as f64 / n; + let z2 = z * z; + let denominator = 1.0 + z2 / n; + let center = (p + z2 / (2.0 * n)) / denominator; + let radius = z + * ((p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt()) + / denominator; + ((center - radius).max(0.0), (center + radius).min(1.0)) +} + +/// One paired nuisance realization inside a generated environment. /// -/// Each run index must represent the same nuisance realization for candidate and -/// control (for example the same representation/learner/stream seed tuple). +/// `nuisance_digest` should identify the representation/learner/stream seed +/// tuple (and any other frozen nuisance settings) shared by candidate/control. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PairedRunResult { + pub nuisance_digest: String, + pub candidate: f64, + pub control: f64, +} + +impl PairedRunResult { + pub fn validate(&self) -> Result<(), String> { + if !looks_like_digest(&self.nuisance_digest) { + return Err("nuisance-run digest must be a 32-byte hex digest".into()); + } + if !self.candidate.is_finite() || !self.control.is_finite() { + return Err("paired run outcomes must be finite".into()); + } + Ok(()) + } + + pub fn delta(&self) -> f64 { + self.candidate - self.control + } +} + +/// Candidate/control outcomes nested within one independently generated world. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct NestedEnvironmentResult { pub environment_digest: String, - pub candidate_runs: Vec, - pub control_runs: Vec, + pub paired_runs: Vec, } impl NestedEnvironmentResult { @@ -49,30 +97,25 @@ impl NestedEnvironmentResult { if !looks_like_digest(&self.environment_digest) { return Err("environment digest must be a 32-byte hex digest".into()); } - if self.candidate_runs.is_empty() || self.candidate_runs.len() != self.control_runs.len() { - return Err("candidate/control nested runs must be non-empty and paired".into()); + if self.paired_runs.is_empty() { + return Err("at least one paired nuisance run is required".into()); } - if self - .candidate_runs - .iter() - .chain(&self.control_runs) - .any(|value| !value.is_finite()) - { - return Err("nested run outcomes must be finite".into()); + let mut nuisance_ids = BTreeSet::new(); + for run in &self.paired_runs { + run.validate()?; + if !nuisance_ids.insert(run.nuisance_digest.to_ascii_lowercase()) { + return Err("duplicate nuisance-run digest within one environment".into()); + } } Ok(()) } pub fn run_count(&self) -> usize { - self.candidate_runs.len() + self.paired_runs.len() } pub fn mean_delta(&self) -> f64 { - self.candidate_runs - .iter() - .zip(&self.control_runs) - .map(|(candidate, control)| candidate - control) - .sum::() + self.paired_runs.iter().map(PairedRunResult::delta).sum::() / self.run_count() as f64 } } @@ -94,10 +137,13 @@ fn validate_environment_results(results: &[NestedEnvironmentResult]) -> Result<( /// Equal-environment-weight hierarchical bootstrap estimate. /// /// The bootstrap first samples environments with replacement. For each sampled -/// environment it then samples paired nested runs with replacement and computes +/// environment it then samples paired nuisance runs with replacement and computes /// one environment mean delta. The final replicate is the equal-weight mean of /// sampled environment means, so environments with more nuisance runs do not /// receive more scientific weight. +/// +/// v1 intentionally reports a percentile interval rather than calling this BCa: +/// the existing one-level BCa helper is not silently repurposed for nested data. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct HierarchicalEstimate { pub n_environments: usize, @@ -135,8 +181,8 @@ pub fn hierarchical_environment_delta_percentile( let environment = &results[rng.gen_range(0..n_environments)]; let mut nested_sum = 0.0; for _ in 0..environment.run_count() { - let run = rng.gen_range(0..environment.run_count()); - nested_sum += environment.candidate_runs[run] - environment.control_runs[run]; + let run = &environment.paired_runs[rng.gen_range(0..environment.run_count())]; + nested_sum += run.delta(); } environment_sum += nested_sum / environment.run_count() as f64; } @@ -165,11 +211,11 @@ pub enum PowerDirection { pub struct ProspectivePowerConfig { /// Candidate numbers of independent environments, evaluated in ascending order. pub environment_counts: Vec, - /// Number of nested paired runs planned per future environment. + /// Number of paired nuisance runs planned per future environment. pub runs_per_environment: usize, /// Monte Carlo future-study simulations per candidate environment count. pub simulation_trials: usize, - /// BCa resamples used inside each simulated future study. + /// BCa resamples used across environment aggregates in each future study. pub bootstrap_resamples: usize, /// Desired probability of clearing the practical-effect gate. pub target_power: f64, @@ -215,14 +261,22 @@ pub struct PowerPoint { pub successes: usize, pub simulation_trials: usize, pub estimated_power: f64, + pub power_ci95_low: f64, + pub power_ci95_high: f64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ProspectivePowerPlan { + /// Digest of the exact DEV experiment manifest used for planning. + pub dev_manifest_digest: String, + /// Digest over DEV nested outcomes plus all power-planning settings. + pub planning_input_digest: String, pub target_power: f64, pub sesoi: f64, pub direction: PowerDirection, pub runs_per_environment: usize, + /// First tested count whose lower power bound clears target and remains clear + /// for every larger tested count. `None` means the tested grid was insufficient. pub minimum_environments: Option, pub points: Vec, } @@ -231,21 +285,37 @@ pub struct ProspectivePowerPlan { /// /// This is an empirical Monte Carlo planning tool, not an analytic guarantee. /// Each future environment is sampled from the DEV environment distribution, -/// then nested paired runs are resampled within that environment. The simulated +/// then paired nuisance runs are resampled within that environment. The simulated /// study is counted as successful only when its environment-level BCa interval /// satisfies the same SESOI gate used for practical-effect interpretation. /// -/// Freeze the chosen sample size before observing CONFIRM outcomes. +/// The selected count is conservative in two ways: the lower Wilson 95% bound on +/// Monte Carlo power must clear `target_power`, and the crossing must remain clear +/// for every larger candidate count tested. Freeze the resulting plan before +/// observing CONFIRM outcomes. pub fn prospective_power_from_dev( + dev_manifest: &ExperimentManifest, dev_results: &[NestedEnvironmentResult], config: &ProspectivePowerConfig, ) -> Result { + dev_manifest.validate()?; + if dev_manifest.stream_namespace != StreamNamespace::Dev + || dev_manifest.tuning_status != TuningStatus::Exploratory + { + return Err("prospective power planning must use an exploratory DEV manifest".into()); + } validate_environment_results(dev_results)?; config.validate()?; + let dev_manifest_digest = dev_manifest.digest().map_err(|error| error.to_string())?; + let planning_input_digest = digest_serialized(&( + dev_manifest_digest.as_str(), + dev_results, + config, + ))?; + let mut rng = StdRng::seed_from_u64(config.seed); let mut points = Vec::with_capacity(config.environment_counts.len()); - let mut minimum_environments = None; for &environment_count in &config.environment_counts { let mut successes = 0usize; @@ -256,8 +326,8 @@ pub fn prospective_power_from_dev( let template = &dev_results[rng.gen_range(0..dev_results.len())]; let mut delta_sum = 0.0; for _ in 0..config.runs_per_environment { - let run = rng.gen_range(0..template.run_count()); - delta_sum += template.candidate_runs[run] - template.control_runs[run]; + let run = &template.paired_runs[rng.gen_range(0..template.run_count())]; + delta_sum += run.delta(); } future_environment_means.push(delta_sum / config.runs_per_environment as f64); } @@ -292,18 +362,28 @@ pub fn prospective_power_from_dev( } let estimated_power = successes as f64 / config.simulation_trials as f64; - if minimum_environments.is_none() && estimated_power >= config.target_power { - minimum_environments = Some(environment_count); - } + let (power_ci95_low, power_ci95_high) = + wilson_interval(successes, config.simulation_trials); points.push(PowerPoint { environments: environment_count, successes, simulation_trials: config.simulation_trials, estimated_power, + power_ci95_low, + power_ci95_high, }); } + let minimum_environments = points.iter().enumerate().find_map(|(index, point)| { + let sustained = points[index..] + .iter() + .all(|later| later.power_ci95_low >= config.target_power); + sustained.then_some(point.environments) + }); + Ok(ProspectivePowerPlan { + dev_manifest_digest, + planning_input_digest, target_power: config.target_power, sesoi: config.sesoi, direction: config.direction, @@ -316,16 +396,53 @@ pub fn prospective_power_from_dev( #[cfg(test)] mod tests { use super::*; + use crate::experiment::{ + EXPERIMENT_MANIFEST_SCHEMA_V1, SeedManifest, + }; fn digest(value: u64) -> String { format!("{value:064x}") } + fn run(id: u64, delta: f64) -> PairedRunResult { + PairedRunResult { + nuisance_digest: digest(id), + candidate: 0.5 + delta, + control: 0.5, + } + } + fn environment(id: u64, deltas: &[f64]) -> NestedEnvironmentResult { NestedEnvironmentResult { - environment_digest: digest(id), - candidate_runs: deltas.iter().map(|delta| 0.5 + delta).collect(), - control_runs: vec![0.5; deltas.len()], + environment_digest: digest(10_000 + id), + paired_runs: deltas + .iter() + .enumerate() + .map(|(index, delta)| run(id * 100 + index as u64 + 1, *delta)) + .collect(), + } + } + + fn dev_manifest() -> ExperimentManifest { + ExperimentManifest { + schema: EXPERIMENT_MANIFEST_SCHEMA_V1.into(), + experiment_id: "SYM-ARCH-002A2-DEV".into(), + experiment_version: "v1".into(), + code_revision: "deadbeef".into(), + preregistration_hash: digest(1), + generator_hash: digest(2), + stream_namespace: StreamNamespace::Dev, + tuning_status: TuningStatus::Exploratory, + prior_results_observed: true, + seed_manifest: SeedManifest { + environment_seeds: vec![1, 2, 3], + representation_seeds: vec![11, 12], + learner_seeds: vec![21, 22], + stream_seeds: vec![31, 32], + }, + primary_hypothesis: "DEV effect distribution informs prospective power".into(), + primary_comparator: "matched control".into(), + sesoi: 0.05, } } @@ -340,8 +457,9 @@ mod tests { assert_eq!(estimate.n_environments, 3); assert_eq!(estimate.total_nested_pairs, 13); assert!((estimate.mean_delta - 0.10).abs() < 1e-12); - assert!(estimate.ci95_low <= estimate.mean_delta); - assert!(estimate.ci95_high >= estimate.mean_delta); + assert!(estimate.ci95_low.is_finite()); + assert!(estimate.ci95_high.is_finite()); + assert!(estimate.ci95_low <= estimate.ci95_high); } #[test] @@ -359,16 +477,22 @@ mod tests { #[test] fn hierarchical_estimate_rejects_duplicate_environment_identity() { - let results = vec![ - environment(1, &[0.1]), - environment(1, &[0.2]), - environment(2, &[0.3]), - ]; + let first = environment(1, &[0.1]); + let mut duplicate = environment(2, &[0.2]); + duplicate.environment_digest = first.environment_digest.clone(); + let results = vec![first, duplicate, environment(3, &[0.3])]; assert!(hierarchical_environment_delta_percentile(&results, 500, 42).is_err()); } #[test] - fn prospective_power_plan_is_deterministic_and_finds_strong_effect() { + fn nested_environment_rejects_duplicate_nuisance_identity() { + let mut result = environment(1, &[0.1, 0.2]); + result.paired_runs[1].nuisance_digest = result.paired_runs[0].nuisance_digest.clone(); + assert!(result.validate().is_err()); + } + + #[test] + fn prospective_power_plan_is_deterministic_bound_and_finds_strong_effect() { let dev = vec![ environment(1, &[0.11, 0.12, 0.13]), environment(2, &[0.10, 0.12, 0.14]), @@ -387,12 +511,39 @@ mod tests { direction: PowerDirection::MeaningfulGain, seed: 99, }; - let first = prospective_power_from_dev(&dev, &config).unwrap(); - let second = prospective_power_from_dev(&dev, &config).unwrap(); + let manifest = dev_manifest(); + let first = prospective_power_from_dev(&manifest, &dev, &config).unwrap(); + let second = prospective_power_from_dev(&manifest, &dev, &config).unwrap(); assert_eq!(first, second); assert_eq!(first.points.len(), 3); + assert!(looks_like_digest(&first.dev_manifest_digest)); + assert!(looks_like_digest(&first.planning_input_digest)); assert!(first.minimum_environments.is_some()); - assert!(first.points.iter().all(|point| (0.0..=1.0).contains(&point.estimated_power))); + assert!(first.points.iter().all(|point| { + (0.0..=1.0).contains(&point.estimated_power) + && point.power_ci95_low <= point.estimated_power + && point.estimated_power <= point.power_ci95_high + })); + } + + #[test] + fn prospective_power_rejects_non_dev_manifest() { + let dev = vec![environment(1, &[0.1]), environment(2, &[0.1]), environment(3, &[0.1])]; + let config = ProspectivePowerConfig { + environment_counts: vec![3], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + let mut manifest = dev_manifest(); + manifest.stream_namespace = StreamNamespace::Confirm; + manifest.tuning_status = TuningStatus::ConfirmatoryFirstUse; + manifest.prior_results_observed = false; + assert!(prospective_power_from_dev(&manifest, &dev, &config).is_err()); } #[test] From e1aa37b7179cef7b0a81a8474de01a39916342ff Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:04:36 +0200 Subject: [PATCH 03/13] feat(research): expose hierarchical experiment statistics --- crates/domains/symthaea-psych-bench/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index df0a8b5ea..512b087dd 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -56,6 +56,8 @@ pub mod benchmarks; pub mod experiment; #[path = "experiment/confirmatory.rs"] pub mod experiment_confirmatory; +#[path = "experiment/statistics.rs"] +pub mod experiment_statistics; pub mod harness; pub mod substrate_transfer; pub mod wm; From 2b6e02450c11475773369df658cd1328b2a4bef3 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:06:18 +0200 Subject: [PATCH 04/13] fix(research): anchor power planning away from DEV winner bias --- .../src/experiment/statistics.rs | 162 +++++++++++++++--- 1 file changed, 134 insertions(+), 28 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs index 6071056e3..f6ca91274 100644 --- a/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs +++ b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs @@ -52,9 +52,7 @@ fn wilson_interval(successes: usize, trials: usize) -> (f64, f64) { let z2 = z * z; let denominator = 1.0 + z2 / n; let center = (p + z2 / (2.0 * n)) / denominator; - let radius = z - * ((p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt()) - / denominator; + let radius = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt() / denominator; ((center - radius).max(0.0), (center + radius).min(1.0)) } @@ -221,6 +219,12 @@ pub struct ProspectivePowerConfig { pub target_power: f64, /// Frozen smallest effect size of interest in the primary metric's natural units. pub sesoi: f64, + /// Frozen alternative effect used for planning. This is deliberately separate + /// from the observed DEV mean to avoid winner's-curse sample-size selection. + pub planning_effect: f64, + /// Multiplier applied to DEV residuals around their equal-environment mean. + /// Values >= 1.0 allow conservative variance-inflation sensitivity analysis. + pub residual_scale: f64, pub direction: PowerDirection, pub seed: u64, } @@ -251,6 +255,21 @@ impl ProspectivePowerConfig { if !self.sesoi.is_finite() || self.sesoi <= 0.0 { return Err("SESOI must be finite and strictly positive".into()); } + if !self.planning_effect.is_finite() { + return Err("planning effect must be finite".into()); + } + match self.direction { + PowerDirection::MeaningfulGain if self.planning_effect <= self.sesoi => { + return Err("gain planning effect must be strictly greater than +SESOI".into()); + } + PowerDirection::MeaningfulRegression if self.planning_effect >= -self.sesoi => { + return Err("regression planning effect must be strictly below -SESOI".into()); + } + _ => {} + } + if !self.residual_scale.is_finite() || self.residual_scale < 1.0 { + return Err("residual_scale must be finite and at least 1.0".into()); + } Ok(()) } } @@ -273,6 +292,8 @@ pub struct ProspectivePowerPlan { pub planning_input_digest: String, pub target_power: f64, pub sesoi: f64, + pub planning_effect: f64, + pub residual_scale: f64, pub direction: PowerDirection, pub runs_per_environment: usize, /// First tested count whose lower power bound clears target and remains clear @@ -281,13 +302,19 @@ pub struct ProspectivePowerPlan { pub points: Vec, } -/// Estimate a future CONFIRM sample size using DEV outcomes only. +/// Estimate a future CONFIRM sample size using DEV variability only. /// /// This is an empirical Monte Carlo planning tool, not an analytic guarantee. +/// DEV supplies the empirical environment/nuisance residual distribution, but +/// simulated effects are re-centered on the separately frozen `planning_effect` +/// rather than on the observed DEV mean. This avoids using a potentially tuned +/// or winner's-curse DEV point estimate as the alternative hypothesis. +/// /// Each future environment is sampled from the DEV environment distribution, -/// then paired nuisance runs are resampled within that environment. The simulated -/// study is counted as successful only when its environment-level BCa interval -/// satisfies the same SESOI gate used for practical-effect interpretation. +/// paired nuisance runs are resampled within that environment, and residuals are +/// optionally inflated by `residual_scale`. A simulated study is successful only +/// when its environment-level BCa interval satisfies the same SESOI gate used for +/// practical-effect interpretation. /// /// The selected count is conservative in two ways: the lower Wilson 95% bound on /// Monte Carlo power must clear `target_power`, and the crossing must remain clear @@ -306,13 +333,18 @@ pub fn prospective_power_from_dev( } validate_environment_results(dev_results)?; config.validate()?; + if dev_results.len() != dev_manifest.seed_manifest.environment_seeds.len() { + return Err("DEV result count must match the frozen environment-seed manifest".into()); + } let dev_manifest_digest = dev_manifest.digest().map_err(|error| error.to_string())?; - let planning_input_digest = digest_serialized(&( - dev_manifest_digest.as_str(), - dev_results, - config, - ))?; + let planning_input_digest = + digest_serialized(&(dev_manifest_digest.as_str(), dev_results, config))?; + let dev_mean = dev_results + .iter() + .map(NestedEnvironmentResult::mean_delta) + .sum::() + / dev_results.len() as f64; let mut rng = StdRng::seed_from_u64(config.seed); let mut points = Vec::with_capacity(config.environment_counts.len()); @@ -327,7 +359,8 @@ pub fn prospective_power_from_dev( let mut delta_sum = 0.0; for _ in 0..config.runs_per_environment { let run = &template.paired_runs[rng.gen_range(0..template.run_count())]; - delta_sum += run.delta(); + let residual = run.delta() - dev_mean; + delta_sum += config.planning_effect + config.residual_scale * residual; } future_environment_means.push(delta_sum / config.runs_per_environment as f64); } @@ -386,6 +419,8 @@ pub fn prospective_power_from_dev( planning_input_digest, target_power: config.target_power, sesoi: config.sesoi, + planning_effect: config.planning_effect, + residual_scale: config.residual_scale, direction: config.direction, runs_per_environment: config.runs_per_environment, minimum_environments, @@ -396,9 +431,7 @@ pub fn prospective_power_from_dev( #[cfg(test)] mod tests { use super::*; - use crate::experiment::{ - EXPERIMENT_MANIFEST_SCHEMA_V1, SeedManifest, - }; + use crate::experiment::{EXPERIMENT_MANIFEST_SCHEMA_V1, SeedManifest}; fn digest(value: u64) -> String { format!("{value:064x}") @@ -423,7 +456,7 @@ mod tests { } } - fn dev_manifest() -> ExperimentManifest { + fn dev_manifest(environment_count: usize) -> ExperimentManifest { ExperimentManifest { schema: EXPERIMENT_MANIFEST_SCHEMA_V1.into(), experiment_id: "SYM-ARCH-002A2-DEV".into(), @@ -435,12 +468,12 @@ mod tests { tuning_status: TuningStatus::Exploratory, prior_results_observed: true, seed_manifest: SeedManifest { - environment_seeds: vec![1, 2, 3], + environment_seeds: (1..=environment_count as u64).collect(), representation_seeds: vec![11, 12], learner_seeds: vec![21, 22], stream_seeds: vec![31, 32], }, - primary_hypothesis: "DEV effect distribution informs prospective power".into(), + primary_hypothesis: "DEV variability informs prospective power".into(), primary_comparator: "matched control".into(), sesoi: 0.05, } @@ -494,12 +527,12 @@ mod tests { #[test] fn prospective_power_plan_is_deterministic_bound_and_finds_strong_effect() { let dev = vec![ - environment(1, &[0.11, 0.12, 0.13]), - environment(2, &[0.10, 0.12, 0.14]), - environment(3, &[0.12, 0.13, 0.15]), - environment(4, &[0.09, 0.11, 0.13]), - environment(5, &[0.11, 0.13, 0.14]), - environment(6, &[0.10, 0.12, 0.13]), + environment(1, &[0.01, 0.02, 0.03]), + environment(2, &[-0.01, 0.02, 0.04]), + environment(3, &[0.02, 0.03, 0.05]), + environment(4, &[-0.02, 0.01, 0.03]), + environment(5, &[0.01, 0.03, 0.04]), + environment(6, &[0.00, 0.02, 0.03]), ]; let config = ProspectivePowerConfig { environment_counts: vec![3, 5, 8], @@ -508,10 +541,12 @@ mod tests { bootstrap_resamples: 100, target_power: 0.80, sesoi: 0.05, + planning_effect: 0.12, + residual_scale: 1.0, direction: PowerDirection::MeaningfulGain, seed: 99, }; - let manifest = dev_manifest(); + let manifest = dev_manifest(dev.len()); let first = prospective_power_from_dev(&manifest, &dev, &config).unwrap(); let second = prospective_power_from_dev(&manifest, &dev, &config).unwrap(); assert_eq!(first, second); @@ -526,9 +561,37 @@ mod tests { })); } + #[test] + fn prospective_power_does_not_use_observed_dev_mean_as_planning_effect() { + let dev = vec![ + environment(1, &[0.30]), + environment(2, &[0.30]), + environment(3, &[0.30]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.08, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 5, + }; + let plan = prospective_power_from_dev(&dev_manifest(3), &dev, &config).unwrap(); + assert_eq!(plan.planning_effect, 0.08); + assert_eq!(plan.points[0].estimated_power, 1.0); + } + #[test] fn prospective_power_rejects_non_dev_manifest() { - let dev = vec![environment(1, &[0.1]), environment(2, &[0.1]), environment(3, &[0.1])]; + let dev = vec![ + environment(1, &[0.1]), + environment(2, &[0.1]), + environment(3, &[0.1]), + ]; let config = ProspectivePowerConfig { environment_counts: vec![3], runs_per_environment: 1, @@ -536,16 +599,57 @@ mod tests { bootstrap_resamples: 100, target_power: 0.80, sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, direction: PowerDirection::MeaningfulGain, seed: 1, }; - let mut manifest = dev_manifest(); + let mut manifest = dev_manifest(3); manifest.stream_namespace = StreamNamespace::Confirm; manifest.tuning_status = TuningStatus::ConfirmatoryFirstUse; manifest.prior_results_observed = false; assert!(prospective_power_from_dev(&manifest, &dev, &config).is_err()); } + #[test] + fn prospective_power_rejects_manifest_result_count_mismatch() { + let dev = vec![ + environment(1, &[0.1]), + environment(2, &[0.1]), + environment(3, &[0.1]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + assert!(prospective_power_from_dev(&dev_manifest(4), &dev, &config).is_err()); + } + + #[test] + fn prospective_power_rejects_effect_inside_practical_margin() { + let config = ProspectivePowerConfig { + environment_counts: vec![5, 8], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.04, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + assert!(config.validate().is_err()); + } + #[test] fn prospective_power_rejects_unsorted_environment_counts() { let config = ProspectivePowerConfig { @@ -555,6 +659,8 @@ mod tests { bootstrap_resamples: 100, target_power: 0.80, sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, direction: PowerDirection::MeaningfulGain, seed: 1, }; From 7e2718d86078d5051523b6baf916aac7235c1e19 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:06:58 +0200 Subject: [PATCH 05/13] docs(research): preregister 002A2 hierarchical power contract --- .../SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md diff --git a/docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md b/docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md new file mode 100644 index 000000000..6fde33b7a --- /dev/null +++ b/docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md @@ -0,0 +1,143 @@ +# SYM-ARCH-002A2 Hierarchical Statistics and Prospective Power v1 + +**Status:** statistical-infrastructure tranche +**Scientific claim status:** none +**Parent plan:** issue #55 +**Stacked dependency:** PR #57 (`research/sym-arch-002a-core-v1`) + +## Purpose + +SYM-ARCH-002A2 extends the experimental core with two pieces needed before a future CONFIRM run can be sized or interpreted responsibly: + +1. hierarchical candidate-minus-control uncertainty with **generated environments as the independent unit**; +2. prospective sample-size planning from DEV variability without using the observed DEV mean as the assumed confirmatory effect. + +This tranche does not choose a CONFIRM sample size because no qualifying DEV campaign is part of this PR. It supplies the instrument that will make that choice later from a frozen DEV artifact and a separately frozen planning effect. + +## Hierarchical result schema + +`NestedEnvironmentResult` contains: + +- one unique environment digest; +- one or more `PairedRunResult` entries; +- a unique nuisance-run digest for each nested representation/learner/stream realization; +- paired candidate/control outcomes from that same nuisance realization. + +Validation rejects: + +- malformed environment or nuisance digests; +- duplicate environment identities; +- duplicate nuisance-run identities within an environment; +- empty nested-run sets; +- non-finite outcomes. + +Nested runs reduce within-environment uncertainty. They do **not** increase the number of independent environments. + +## Hierarchical bootstrap + +`hierarchical_environment_delta_percentile` performs a two-level paired bootstrap: + +1. sample environments with replacement; +2. inside each sampled environment, sample paired nuisance runs with replacement; +3. compute one mean delta for each sampled environment; +4. average environment means with **equal environment weight**; +5. repeat and report a deterministic 95% percentile interval. + +An environment with ten nuisance runs therefore does not receive ten times the scientific weight of an environment with one nuisance run. + +v1 deliberately calls this a **hierarchical percentile interval**, not BCa. The existing one-level BCa implementation is not silently relabeled as a nested-data method. + +## Prospective power planning + +`prospective_power_from_dev` is allowed only with a valid exploratory `DEV` manifest. The number of DEV environment results must match the frozen environment-seed manifest. + +The planner uses DEV for empirical heterogeneity/noise, but **not for the assumed effect size**. This prevents a tuned or unusually favorable DEV point estimate from becoming a winner's-curse input to CONFIRM sizing. + +The preregistered planning configuration includes: + +- candidate independent-environment counts; +- nested paired runs planned per future environment; +- Monte Carlo trial count; +- BCa resamples across future environment aggregates; +- target power; +- SESOI; +- a separately frozen `planning_effect`; +- a residual-variance scale; +- gain/regression direction; +- RNG seed. + +For a gain claim, `planning_effect` must be strictly greater than `+SESOI`. For a regression-detection plan it must be strictly below `-SESOI`. + +### DEV winner-bias guard + +Let `d_dev` be the equal-environment DEV mean delta. A sampled DEV run contributes a residual: + +`residual = run_delta - d_dev` + +The simulated future delta is: + +`planning_effect + residual_scale * residual` + +Thus DEV contributes the observed environment/nuisance variability while the future-study center remains the independently frozen planning effect. + +`residual_scale >= 1` supports conservative sensitivity analysis. A value below one is rejected because it would make the observed DEV variability artificially easier. + +## Practical-effect-aligned power criterion + +A simulated future study counts as successful only when its environment-level 95% BCa interval clears the same SESOI practical-effect gate used by 002A1. + +Power is therefore not defined as merely obtaining `p < 0.05`. It is the probability that the future study will support the **predeclared practically meaningful claim**. + +For each candidate environment count, A2 reports: + +- Monte Carlo success count; +- point estimate of power; +- Wilson 95% interval for that estimated power. + +The selected `minimum_environments`, if any, is the first tested count whose **lower Wilson power bound** clears the target **and remains above target for every larger tested count**. This prevents a single noisy threshold crossing from setting CONFIRM size. + +If no tested count satisfies this sustained conservative rule, `minimum_environments = None`; the correct response is to extend the planning grid or reconsider the design, not to weaken the target after looking at results. + +## Provenance binding + +The power plan records: + +- digest of the exact DEV experiment manifest; +- digest over the DEV nested outcomes and complete power configuration. + +Changing DEV data, planning effect, SESOI, variance inflation, candidate counts, simulation settings, or seed therefore changes the planning-input digest. + +The chosen CONFIRM sample size should later be frozen by referencing this digest before CONFIRM outcomes are observed. + +## Acceptance gate + +The exact stacked PR head should pass: + +1. rustfmt on A2 Rust paths; +2. focused `experiment_statistics` tests; +3. `cargo check -p symthaea-psych-bench --lib`; +4. equal-environment-weight reference test; +5. deterministic hierarchical-bootstrap test; +6. duplicate environment/nuisance identity rejection; +7. DEV-only planning rejection for non-DEV manifests; +8. manifest/result environment-count binding; +9. planning-effect-outside-SESOI validation; +10. deterministic power-plan/provenance-digest test; +11. Wilson-bounded power reporting and sustained-threshold selection. + +## Explicit non-claims and limitations + +A2 v1 does **not** claim: + +- that percentile hierarchical intervals are optimal for every future metric; +- that the empirical DEV variance distribution perfectly represents CONFIRM; +- that a planning effect should be chosen from DEV performance; +- that any particular number of environments is currently required; +- that 002 has demonstrated an architecture advantage; +- that multiple-comparison correction, resource Pareto analysis, or procedural benchmark validity is complete. + +The `planning_effect` must be justified and frozen outside this code path. Sensitivity analysis should run multiple preregistered `residual_scale >= 1` conditions where DEV is small or heterogeneous. + +## Merge boundary + +This PR is statistical infrastructure and may merge on correctness after its exact-head gate passes. It should remain stacked on #57 until #57 merges; after that, rebase/retarget to `main` without changing the statistical contract. From f949c00d1eb06a9c14fac73a5ddea4ad77205b50 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:07:11 +0200 Subject: [PATCH 06/13] ci(research): gate 002A2 hierarchical power infrastructure --- .../sym-arch-002a2-hierarchical-power.yml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/sym-arch-002a2-hierarchical-power.yml diff --git a/.github/workflows/sym-arch-002a2-hierarchical-power.yml b/.github/workflows/sym-arch-002a2-hierarchical-power.yml new file mode 100644 index 000000000..ae9097abc --- /dev/null +++ b/.github/workflows/sym-arch-002a2-hierarchical-power.yml @@ -0,0 +1,65 @@ +name: SYM-ARCH-002A2 Hierarchical Power + +on: + pull_request: + branches: + - main + - research/sym-arch-002a-core-v1 + paths: + - '.github/workflows/sym-arch-002a2-hierarchical-power.yml' + - 'crates/domains/symthaea-psych-bench/src/experiment/statistics.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/confirmatory.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/mod.rs' + - 'crates/domains/symthaea-psych-bench/src/lib.rs' + - 'crates/domains/symthaea-psych-bench/src/harness/analysis.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' + - 'docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + hierarchical-power: + name: Validate hierarchical statistics and power planning + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + with: + components: rustfmt + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-002a2-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-002a2- + + - name: Check A2 Rust formatting + run: | + rustfmt --edition 2024 --check \ + crates/domains/symthaea-psych-bench/src/experiment/statistics.rs \ + crates/domains/symthaea-psych-bench/src/lib.rs + + - name: Run hierarchical statistics and power tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_statistics:: -- --nocapture + + - name: Check psych-bench library + run: | + cargo check -p symthaea-psych-bench --lib From d7f6f681e08c95f10a25b24d8acae4c348886f78 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:55:04 +0200 Subject: [PATCH 07/13] fix(test): remove stale loop trial fixture field --- .../symthaea-psych-bench/src/harness/neuromod_correlation.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs index b4c849fdc..c6056aad3 100644 --- a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs +++ b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs @@ -241,11 +241,6 @@ mod tests { cycle_time_us: 1000 + (i as u64 * 50), learning_occurred: i % 5 == 0, reward: 0.0, - // Added 2026-07-31 to unblock the crate: another session's - // LoopTrialResult gained `cycle_reward` and this test fixture was - // collateral damage. Synthetic data, so 0.0 matches the - // neighbouring `reward`/`moral_score` fixtures. - cycle_reward: 0.0, oxytocin: 0.3, moral_score: 0.0, bath_entropy: 0.5, From 63733e53528ca1fc59c6e969edf3839cd3f0c3da Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 12:57:32 +0200 Subject: [PATCH 08/13] feat(research): distinguish nested and crossed nuisance designs --- .../src/experiment/statistics_design.rs | 360 ++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs diff --git a/crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs b/crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs new file mode 100644 index 000000000..942660760 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs @@ -0,0 +1,360 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Nuisance-topology guards for SYM-ARCH-002A2. +//! +//! A representation/learner/stream realization may be genuinely nested inside +//! one generated environment, or the same realization may be reused across many +//! environments. Those designs have different dependence structures and must not +//! share an uncertainty procedure silently. +//! +//! This module is the fail-closed scientific entry point around the lower-level +//! v1 statistics implementation: +//! +//! - `NestedIndependent` requires nuisance identities to be globally unique; +//! - `CrossedShared` requires the same complete nuisance grid in every environment; +//! - crossed uncertainty uses a two-way environment × nuisance bootstrap; +//! - v1 prospective power refuses crossed designs until a crossed power simulator +//! is implemented and separately validated. + +use crate::experiment::{ExperimentManifest, PairedEstimate}; +use crate::experiment_statistics::{ + NestedEnvironmentResult, ProspectivePowerConfig, ProspectivePowerPlan, + prospective_power_from_dev, +}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NuisanceTopology { + /// Every nuisance realization belongs to exactly one environment. + NestedIndependent, + /// The same nuisance identities form a complete balanced grid in every environment. + CrossedShared, +} + +fn normalized_environment_grid( + results: &[NestedEnvironmentResult], +) -> Result>, String> { + if results.len() < 3 { + return Err("at least three independent environments are required".into()); + } + + let mut environment_ids = BTreeSet::new(); + let mut grids = Vec::with_capacity(results.len()); + for environment in results { + environment.validate()?; + let environment_id = environment.environment_digest.to_ascii_lowercase(); + if !environment_ids.insert(environment_id) { + return Err("duplicate environment digest would create pseudoreplication".into()); + } + + let mut grid = BTreeMap::new(); + for run in &environment.paired_runs { + let nuisance_id = run.nuisance_digest.to_ascii_lowercase(); + if grid.insert(nuisance_id, run.delta()).is_some() { + return Err("duplicate nuisance identity within one environment".into()); + } + } + grids.push(grid); + } + Ok(grids) +} + +/// Validate the dependence topology before scientific inference. +/// +/// This is intentionally stricter than `NestedEnvironmentResult::validate`, which +/// can validate only one environment at a time and therefore cannot know whether +/// a nuisance realization has been reused across environments. +pub fn validate_nuisance_topology( + results: &[NestedEnvironmentResult], + topology: NuisanceTopology, +) -> Result<(), String> { + let grids = normalized_environment_grid(results)?; + + match topology { + NuisanceTopology::NestedIndependent => { + let mut globally_seen = BTreeSet::new(); + for grid in &grids { + for nuisance_id in grid.keys() { + if !globally_seen.insert(nuisance_id.clone()) { + return Err( + "nested-independent nuisance design reuses a nuisance identity across environments" + .into(), + ); + } + } + } + } + NuisanceTopology::CrossedShared => { + let reference: BTreeSet<&str> = grids[0].keys().map(String::as_str).collect(); + if reference.len() < 2 { + return Err( + "crossed-shared inference requires at least two nuisance identities".into(), + ); + } + for grid in grids.iter().skip(1) { + let observed: BTreeSet<&str> = grid.keys().map(String::as_str).collect(); + if observed != reference { + return Err( + "crossed-shared nuisance design requires the same complete nuisance grid in every environment" + .into(), + ); + } + } + } + } + + Ok(()) +} + +fn percentile(sorted: &[f64], probability: f64) -> f64 { + debug_assert!(!sorted.is_empty()); + let position = probability.clamp(0.0, 1.0) * (sorted.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + sorted[lower] + } else { + let weight = position - lower as f64; + sorted[lower] * (1.0 - weight) + sorted[upper] * weight + } +} + +/// Equal-cell estimate with a two-way crossed environment × nuisance bootstrap. +/// +/// Each bootstrap replicate independently resamples environment identities and +/// nuisance identities with replacement, then evaluates the Cartesian product of +/// those sampled clusters. Reusing a nuisance draw across all sampled environments +/// preserves the crossed dependence that would be destroyed by independently +/// resampling nuisance runs inside each environment. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CrossedHierarchicalEstimate { + pub n_environments: usize, + pub n_nuisance_identities: usize, + pub total_cells: usize, + pub mean_delta: f64, + pub ci95_low: f64, + pub ci95_high: f64, + pub bootstrap_resamples: usize, +} + +pub fn crossed_environment_nuisance_delta_percentile( + results: &[NestedEnvironmentResult], + n_resamples: usize, + seed: u64, +) -> Result { + validate_nuisance_topology(results, NuisanceTopology::CrossedShared)?; + if n_resamples < 200 { + return Err("crossed hierarchical bootstrap requires at least 200 resamples".into()); + } + + let grids = normalized_environment_grid(results)?; + let nuisance_ids: Vec = grids[0].keys().cloned().collect(); + let n_environments = grids.len(); + let n_nuisance_identities = nuisance_ids.len(); + let total_cells = n_environments + .checked_mul(n_nuisance_identities) + .ok_or_else(|| "crossed cell count overflow".to_string())?; + + let observed_sum: f64 = grids + .iter() + .flat_map(|grid| nuisance_ids.iter().map(move |id| grid[id])) + .sum(); + let mean_delta = observed_sum / total_cells as f64; + + let mut rng = StdRng::seed_from_u64(seed); + let mut bootstrap = Vec::with_capacity(n_resamples); + for _ in 0..n_resamples { + let sampled_environments: Vec = (0..n_environments) + .map(|_| rng.gen_range(0..n_environments)) + .collect(); + let sampled_nuisance: Vec = (0..n_nuisance_identities) + .map(|_| rng.gen_range(0..n_nuisance_identities)) + .collect(); + + let mut sum = 0.0; + for &environment_index in &sampled_environments { + let grid = &grids[environment_index]; + for &nuisance_index in &sampled_nuisance { + sum += grid[&nuisance_ids[nuisance_index]]; + } + } + bootstrap.push(sum / total_cells as f64); + } + + bootstrap.sort_by(|left, right| left.total_cmp(right)); + Ok(CrossedHierarchicalEstimate { + n_environments, + n_nuisance_identities, + total_cells, + mean_delta, + ci95_low: percentile(&bootstrap, 0.025), + ci95_high: percentile(&bootstrap, 0.975), + bootstrap_resamples: n_resamples, + }) +} + +/// Convert a crossed estimate into the common paired-estimate shape when only the +/// mean and interval are needed by downstream SESOI classification. +pub fn crossed_as_paired_estimate(estimate: &CrossedHierarchicalEstimate) -> PairedEstimate { + PairedEstimate { + n_pairs: estimate.n_environments, + mean_delta: estimate.mean_delta, + ci95_low: estimate.ci95_low, + ci95_high: estimate.ci95_high, + } +} + +/// Topology-checked entry point for v1 prospective power planning. +/// +/// The existing power simulator is valid only for genuinely nested nuisance runs. +/// Crossed designs fail closed here rather than silently receiving the nested +/// simulator. A dedicated crossed power simulator should be a separately reviewed +/// follow-up because it must reproduce both environment and shared-nuisance cluster +/// uncertainty inside each simulated study. +pub fn prospective_power_from_dev_topology_checked( + dev_manifest: &ExperimentManifest, + dev_results: &[NestedEnvironmentResult], + config: &ProspectivePowerConfig, + topology: NuisanceTopology, +) -> Result { + validate_nuisance_topology(dev_results, topology)?; + match topology { + NuisanceTopology::NestedIndependent => { + prospective_power_from_dev(dev_manifest, dev_results, config) + } + NuisanceTopology::CrossedShared => Err( + "v1 prospective power does not support crossed nuisance designs; use a separately validated crossed power simulator" + .into(), + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment::{ + EXPERIMENT_MANIFEST_SCHEMA_V1, SeedManifest, StreamNamespace, TuningStatus, + }; + use crate::experiment_statistics::{PairedRunResult, PowerDirection}; + + fn digest(value: u64) -> String { + format!("{value:064x}") + } + + fn environment(id: u64, nuisance: &[(u64, f64)]) -> NestedEnvironmentResult { + NestedEnvironmentResult { + environment_digest: digest(10_000 + id), + paired_runs: nuisance + .iter() + .map(|(nuisance_id, delta)| PairedRunResult { + nuisance_digest: digest(*nuisance_id), + candidate: 0.5 + delta, + control: 0.5, + }) + .collect(), + } + } + + #[test] + fn nested_topology_rejects_cross_environment_nuisance_reuse() { + let results = vec![ + environment(1, &[(1, 0.1)]), + environment(2, &[(1, 0.2)]), + environment(3, &[(3, 0.3)]), + ]; + assert!( + validate_nuisance_topology(&results, NuisanceTopology::NestedIndependent).is_err() + ); + } + + #[test] + fn crossed_topology_requires_a_complete_shared_grid() { + let good = vec![ + environment(1, &[(1, 0.10), (2, 0.20)]), + environment(2, &[(1, 0.00), (2, 0.10)]), + environment(3, &[(1, -0.10), (2, 0.00)]), + ]; + validate_nuisance_topology(&good, NuisanceTopology::CrossedShared).unwrap(); + + let bad = vec![ + environment(1, &[(1, 0.10), (2, 0.20)]), + environment(2, &[(1, 0.00), (3, 0.10)]), + environment(3, &[(1, -0.10), (2, 0.00)]), + ]; + assert!(validate_nuisance_topology(&bad, NuisanceTopology::CrossedShared).is_err()); + } + + #[test] + fn crossed_bootstrap_is_deterministic_and_equal_cell_weighted() { + let results = vec![ + environment(1, &[(1, 0.10), (2, 0.20), (3, 0.30)]), + environment(2, &[(1, 0.00), (2, 0.10), (3, 0.20)]), + environment(3, &[(1, -0.10), (2, 0.00), (3, 0.10)]), + environment(4, &[(1, 0.20), (2, 0.30), (3, 0.40)]), + ]; + let first = crossed_environment_nuisance_delta_percentile(&results, 500, 42).unwrap(); + let second = crossed_environment_nuisance_delta_percentile(&results, 500, 42).unwrap(); + assert_eq!(first, second); + assert_eq!(first.n_environments, 4); + assert_eq!(first.n_nuisance_identities, 3); + assert_eq!(first.total_cells, 12); + assert!((first.mean_delta - 0.15).abs() < 1e-12); + assert!(first.ci95_low <= first.mean_delta); + assert!(first.mean_delta <= first.ci95_high); + } + + #[test] + fn crossed_design_fails_closed_in_v1_power_planner() { + let results = vec![ + environment(1, &[(1, 0.08), (2, 0.12)]), + environment(2, &[(1, 0.06), (2, 0.10)]), + environment(3, &[(1, 0.07), (2, 0.11)]), + ]; + let manifest = ExperimentManifest { + schema: EXPERIMENT_MANIFEST_SCHEMA_V1.into(), + experiment_id: "crossed-power-test".into(), + experiment_version: "v1".into(), + code_revision: "deadbeef".into(), + preregistration_hash: digest(100), + generator_hash: digest(101), + stream_namespace: StreamNamespace::Dev, + tuning_status: TuningStatus::Exploratory, + prior_results_observed: true, + seed_manifest: SeedManifest { + environment_seeds: vec![1, 2, 3], + representation_seeds: vec![1, 2], + learner_seeds: vec![], + stream_seeds: vec![], + }, + primary_hypothesis: "crossed power is fail-closed in v1".into(), + primary_comparator: "matched control".into(), + sesoi: 0.05, + }; + let config = ProspectivePowerConfig { + environment_counts: vec![3, 5], + runs_per_environment: 2, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 7, + }; + + let error = prospective_power_from_dev_topology_checked( + &manifest, + &results, + &config, + NuisanceTopology::CrossedShared, + ) + .unwrap_err(); + assert!(error.contains("does not support crossed nuisance designs")); + } +} From 108619a9d1123c3efe46952d0cac3bc37af934b5 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 12:57:48 +0200 Subject: [PATCH 09/13] feat(research): expose nuisance-topology statistics guards --- crates/domains/symthaea-psych-bench/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index 512b087dd..0e12062ca 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -58,6 +58,8 @@ pub mod experiment; pub mod experiment_confirmatory; #[path = "experiment/statistics.rs"] pub mod experiment_statistics; +#[path = "experiment/statistics_design.rs"] +pub mod experiment_statistics_design; pub mod harness; pub mod substrate_transfer; pub mod wm; @@ -66,4 +68,4 @@ pub mod wm; pub mod neuroevolution_fitness; #[cfg(test)] -mod proptest_math_benchmarks; +mod proptest_math_benchmarks; \ No newline at end of file From 1ed5fd5af79a22aabe242f0eb1fcb08181154d93 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 12:58:07 +0200 Subject: [PATCH 10/13] ci(research): gate nuisance-topology statistics --- .github/workflows/sym-arch-002a2-hierarchical-power.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sym-arch-002a2-hierarchical-power.yml b/.github/workflows/sym-arch-002a2-hierarchical-power.yml index ae9097abc..df85b4068 100644 --- a/.github/workflows/sym-arch-002a2-hierarchical-power.yml +++ b/.github/workflows/sym-arch-002a2-hierarchical-power.yml @@ -8,12 +8,14 @@ on: paths: - '.github/workflows/sym-arch-002a2-hierarchical-power.yml' - 'crates/domains/symthaea-psych-bench/src/experiment/statistics.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs' - 'crates/domains/symthaea-psych-bench/src/experiment/confirmatory.rs' - 'crates/domains/symthaea-psych-bench/src/experiment/mod.rs' - 'crates/domains/symthaea-psych-bench/src/lib.rs' - 'crates/domains/symthaea-psych-bench/src/harness/analysis.rs' - 'crates/domains/symthaea-psych-bench/Cargo.toml' - 'docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md' + - 'docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md' - 'Cargo.toml' - 'Cargo.lock' workflow_dispatch: @@ -54,12 +56,14 @@ jobs: run: | rustfmt --edition 2024 --check \ crates/domains/symthaea-psych-bench/src/experiment/statistics.rs \ + crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs \ crates/domains/symthaea-psych-bench/src/lib.rs - name: Run hierarchical statistics and power tests run: | cargo test -p symthaea-psych-bench --lib experiment_statistics:: -- --nocapture + cargo test -p symthaea-psych-bench --lib experiment_statistics_design:: -- --nocapture - name: Check psych-bench library run: | - cargo check -p symthaea-psych-bench --lib + cargo check -p symthaea-psych-bench --lib \ No newline at end of file From ab669e7216012d28cdcb5c78bed2b2f4dae37dea Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 12:58:30 +0200 Subject: [PATCH 11/13] docs(research): freeze nuisance-topology boundary for A2 --- .../SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md diff --git a/docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md b/docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md new file mode 100644 index 000000000..873e690fb --- /dev/null +++ b/docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md @@ -0,0 +1,82 @@ +# SYM-ARCH-002A2 — Nuisance Topology Boundary v1 + +**Status:** statistical design hardening; no architecture result + +**Tracks:** #55, stacked in #58 + +## Why this exists + +A representation/learner/stream realization can enter an experiment in two materially different ways: + +1. **nested-independent** — each nuisance realization belongs to only one generated environment; +2. **crossed-shared** — the same nuisance realization is reused across multiple generated environments. + +Those observations do not have the same dependence structure. + +If the same representation seed, learner seed, stream ordering, or other nuisance realization is reused across environments, independently resampling nuisance runs inside each environment can treat a shared source of variation as though it were independent. That can distort uncertainty and later power calculations. + +The topology is therefore part of the frozen statistical design, not an implementation detail. + +## `NestedIndependent` + +The scientific guard requires every nuisance digest to be globally unique across environments. + +The existing environment-first + nested-run hierarchical percentile bootstrap is appropriate only after this condition is established. + +A repeated nuisance digest across two environments is rejected instead of being silently treated as nested. + +## `CrossedShared` + +A crossed design must expose the same complete nuisance-identity grid in every environment. Missing or substituted nuisance cells make the design invalid for the v1 crossed estimator. + +A2 now supplies a two-way environment × nuisance percentile bootstrap: + +1. sample environments with replacement; +2. sample nuisance identities with replacement **once per replicate**; +3. apply those sampled nuisance identities across all sampled environments; +4. average the sampled Cartesian product; +5. report the 2.5% and 97.5% bootstrap percentiles. + +Sampling the nuisance identities jointly across environments preserves the fact that a reused nuisance realization is shared rather than independent within each environment. + +The v1 crossed estimator treats the combined representation/learner/stream nuisance digest as one crossed cluster. If later experiments need separate variance components for representation seed, learner seed, and stream seed, that is a distinct multiway-random-effects extension and must not be inferred from this combined digest. + +## Prospective power boundary + +The existing v1 prospective-power simulator was written for nested-independent nuisance runs. + +The topology-checked scientific entry point therefore behaves as follows: + +- `NestedIndependent` → validate global uniqueness, then use the existing DEV-based prospective-power simulator; +- `CrossedShared` → **fail closed** with an explicit unsupported-design error. + +A crossed CONFIRM design must not use the nested power recommendation. + +Before crossed prospective power is allowed, a later implementation must simulate both independent environment sampling and shared nuisance-cluster sampling inside each future study and validate that procedure separately. + +This restriction is intentional. Missing power support is safer than a precise-looking sample-size recommendation from the wrong dependence model. + +## Confirmatory freeze requirements + +Before opening CONFIRM, freeze at least: + +- nuisance topology (`nested_independent` or `crossed_shared`); +- exact definition of the nuisance digest; +- whether representation, learner, and stream seeds are reused across environments; +- environment count; +- nuisance count or runs per environment; +- missing-cell policy (v1 crossed requires none); +- uncertainty procedure; +- prospective-power procedure and version; +- bootstrap seeds/resample counts; +- primary metric, comparator, SESOI, and direction. + +Changing from nested to crossed after observing results creates a new analysis specification; it is not a harmless re-labeling. + +## Claim ceiling + +This hardening supports only: + +> SYM-ARCH-002A2 distinguishes nested-independent from balanced crossed nuisance designs, applies topology-specific uncertainty checks, and refuses to use the v1 nested prospective-power simulator for crossed data. + +It does not establish that either design is preferable, does not choose the eventual CONFIRM topology, and carries no Symthaea capability claim. \ No newline at end of file From 268d66a7a4c71009031112ddde12400d4383eac4 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 15:39:19 +0200 Subject: [PATCH 12/13] ci(research): supersede stale A2 validation runs --- .github/workflows/sym-arch-002a2-hierarchical-power.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sym-arch-002a2-hierarchical-power.yml b/.github/workflows/sym-arch-002a2-hierarchical-power.yml index df85b4068..6499c9569 100644 --- a/.github/workflows/sym-arch-002a2-hierarchical-power.yml +++ b/.github/workflows/sym-arch-002a2-hierarchical-power.yml @@ -20,6 +20,12 @@ on: - 'Cargo.lock' workflow_dispatch: +concurrency: + # Automatic branch runs supersede stale checks. Deliberate manual validation + # runs are unique and therefore never cancel one another. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + permissions: contents: read @@ -66,4 +72,4 @@ jobs: - name: Check psych-bench library run: | - cargo check -p symthaea-psych-bench --lib \ No newline at end of file + cargo check -p symthaea-psych-bench --lib From d76d107fda0aea9751f79ee63a6a89e0fd00ab58 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Wed, 26 Aug 2026 09:28:33 +0200 Subject: [PATCH 13/13] ci(research): defer A2 dedicated gate while draft --- .github/workflows/sym-arch-002a2-hierarchical-power.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sym-arch-002a2-hierarchical-power.yml b/.github/workflows/sym-arch-002a2-hierarchical-power.yml index 6499c9569..5ce0c24bd 100644 --- a/.github/workflows/sym-arch-002a2-hierarchical-power.yml +++ b/.github/workflows/sym-arch-002a2-hierarchical-power.yml @@ -2,6 +2,7 @@ name: SYM-ARCH-002A2 Hierarchical Power on: pull_request: + types: [opened, synchronize, reopened, ready_for_review] branches: - main - research/sym-arch-002a-core-v1 @@ -36,6 +37,9 @@ env: jobs: hierarchical-power: name: Validate hierarchical statistics and power planning + # Draft stacked PRs cannot merge and should not consume scarce runner capacity. + # Promoting to ready-for-review explicitly triggers this exact gate. + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest timeout-minutes: 20