diff --git a/.github/workflows/sym-arch-002a2-hierarchical-power.yml b/.github/workflows/sym-arch-002a2-hierarchical-power.yml new file mode 100644 index 000000000..5ce0c24bd --- /dev/null +++ b/.github/workflows/sym-arch-002a2-hierarchical-power.yml @@ -0,0 +1,79 @@ +name: SYM-ARCH-002A2 Hierarchical Power + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: + - main + - research/sym-arch-002a-core-v1 + paths: + - '.github/workflows/sym-arch-002a2-hierarchical-power.yml' + - 'crates/domains/symthaea-psych-bench/src/experiment/statistics.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/confirmatory.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/mod.rs' + - 'crates/domains/symthaea-psych-bench/src/lib.rs' + - 'crates/domains/symthaea-psych-bench/src/harness/analysis.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' + - 'docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md' + - 'docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +concurrency: + # Automatic branch runs supersede stale checks. Deliberate manual validation + # runs are unique and therefore never cancel one another. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + hierarchical-power: + name: Validate hierarchical statistics and power planning + # Draft stacked PRs cannot merge and should not consume scarce runner capacity. + # Promoting to ready-for-review explicitly triggers this exact gate. + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + with: + components: rustfmt + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-002a2-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-002a2- + + - name: Check A2 Rust formatting + run: | + rustfmt --edition 2024 --check \ + crates/domains/symthaea-psych-bench/src/experiment/statistics.rs \ + crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs \ + crates/domains/symthaea-psych-bench/src/lib.rs + + - name: Run hierarchical statistics and power tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_statistics:: -- --nocapture + cargo test -p symthaea-psych-bench --lib experiment_statistics_design:: -- --nocapture + + - name: Check psych-bench library + run: | + cargo check -p symthaea-psych-bench --lib diff --git a/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs new file mode 100644 index 000000000..f6ca91274 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/statistics.rs @@ -0,0 +1,669 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Hierarchical uncertainty and prospective power planning for architecture research. +//! +//! Confirmatory inference treats independently generated environments as the +//! scientific unit of generalization. Nested representation/learner/stream runs +//! quantify within-environment uncertainty, but do not increase the number of +//! independent environments. + +use crate::experiment::{ExperimentManifest, PairedEstimate, StreamNamespace, TuningStatus}; +use crate::experiment_confirmatory::{PracticalEffect, classify_practical_effect}; +use crate::harness::analysis::bootstrap_ci_bca; +use rand::rngs::StdRng; +use rand::{Rng, RngCore, SeedableRng}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +const POWER_PLAN_INPUT_DOMAIN: &[u8] = b"symthaea.prospective-power.input/v1"; + +fn looks_like_digest(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) +} + +fn percentile(sorted: &[f64], probability: f64) -> f64 { + debug_assert!(!sorted.is_empty()); + let position = probability.clamp(0.0, 1.0) * (sorted.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + sorted[lower] + } else { + let weight = position - lower as f64; + sorted[lower] * (1.0 - weight) + sorted[upper] * weight + } +} + +fn digest_serialized(value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?; + let mut hasher = blake3::Hasher::new(); + hasher.update(POWER_PLAN_INPUT_DOMAIN); + hasher.update(&[0]); + hasher.update(&bytes); + Ok(hasher.finalize().to_hex().to_string()) +} + +fn wilson_interval(successes: usize, trials: usize) -> (f64, f64) { + debug_assert!(trials > 0); + let z = 1.959_963_984_540_054_f64; + let n = trials as f64; + let p = successes as f64 / n; + let z2 = z * z; + let denominator = 1.0 + z2 / n; + let center = (p + z2 / (2.0 * n)) / denominator; + let radius = z * (p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt() / denominator; + ((center - radius).max(0.0), (center + radius).min(1.0)) +} + +/// One paired nuisance realization inside a generated environment. +/// +/// `nuisance_digest` should identify the representation/learner/stream seed +/// tuple (and any other frozen nuisance settings) shared by candidate/control. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PairedRunResult { + pub nuisance_digest: String, + pub candidate: f64, + pub control: f64, +} + +impl PairedRunResult { + pub fn validate(&self) -> Result<(), String> { + if !looks_like_digest(&self.nuisance_digest) { + return Err("nuisance-run digest must be a 32-byte hex digest".into()); + } + if !self.candidate.is_finite() || !self.control.is_finite() { + return Err("paired run outcomes must be finite".into()); + } + Ok(()) + } + + pub fn delta(&self) -> f64 { + self.candidate - self.control + } +} + +/// Candidate/control outcomes nested within one independently generated world. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NestedEnvironmentResult { + pub environment_digest: String, + pub paired_runs: Vec, +} + +impl NestedEnvironmentResult { + pub fn validate(&self) -> Result<(), String> { + if !looks_like_digest(&self.environment_digest) { + return Err("environment digest must be a 32-byte hex digest".into()); + } + if self.paired_runs.is_empty() { + return Err("at least one paired nuisance run is required".into()); + } + let mut nuisance_ids = BTreeSet::new(); + for run in &self.paired_runs { + run.validate()?; + if !nuisance_ids.insert(run.nuisance_digest.to_ascii_lowercase()) { + return Err("duplicate nuisance-run digest within one environment".into()); + } + } + Ok(()) + } + + pub fn run_count(&self) -> usize { + self.paired_runs.len() + } + + pub fn mean_delta(&self) -> f64 { + self.paired_runs.iter().map(PairedRunResult::delta).sum::() + / self.run_count() as f64 + } +} + +fn validate_environment_results(results: &[NestedEnvironmentResult]) -> Result<(), String> { + if results.len() < 3 { + return Err("at least three independent environments are required".into()); + } + let mut seen = BTreeSet::new(); + for result in results { + result.validate()?; + if !seen.insert(result.environment_digest.to_ascii_lowercase()) { + return Err("duplicate environment digest would create pseudoreplication".into()); + } + } + Ok(()) +} + +/// Equal-environment-weight hierarchical bootstrap estimate. +/// +/// The bootstrap first samples environments with replacement. For each sampled +/// environment it then samples paired nuisance runs with replacement and computes +/// one environment mean delta. The final replicate is the equal-weight mean of +/// sampled environment means, so environments with more nuisance runs do not +/// receive more scientific weight. +/// +/// v1 intentionally reports a percentile interval rather than calling this BCa: +/// the existing one-level BCa helper is not silently repurposed for nested data. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct HierarchicalEstimate { + pub n_environments: usize, + pub total_nested_pairs: usize, + pub mean_delta: f64, + pub ci95_low: f64, + pub ci95_high: f64, + pub bootstrap_resamples: usize, +} + +pub fn hierarchical_environment_delta_percentile( + results: &[NestedEnvironmentResult], + n_resamples: usize, + seed: u64, +) -> Result { + validate_environment_results(results)?; + if n_resamples < 200 { + return Err("hierarchical bootstrap requires at least 200 resamples".into()); + } + + let n_environments = results.len(); + let total_nested_pairs = results.iter().map(NestedEnvironmentResult::run_count).sum(); + let mean_delta = results + .iter() + .map(NestedEnvironmentResult::mean_delta) + .sum::() + / n_environments as f64; + + let mut rng = StdRng::seed_from_u64(seed); + let mut bootstrap = Vec::with_capacity(n_resamples); + + for _ in 0..n_resamples { + let mut environment_sum = 0.0; + for _ in 0..n_environments { + let environment = &results[rng.gen_range(0..n_environments)]; + let mut nested_sum = 0.0; + for _ in 0..environment.run_count() { + let run = &environment.paired_runs[rng.gen_range(0..environment.run_count())]; + nested_sum += run.delta(); + } + environment_sum += nested_sum / environment.run_count() as f64; + } + bootstrap.push(environment_sum / n_environments as f64); + } + + bootstrap.sort_by(|left, right| left.total_cmp(right)); + Ok(HierarchicalEstimate { + n_environments, + total_nested_pairs, + mean_delta, + ci95_low: percentile(&bootstrap, 0.025), + ci95_high: percentile(&bootstrap, 0.975), + bootstrap_resamples: n_resamples, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PowerDirection { + MeaningfulGain, + MeaningfulRegression, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProspectivePowerConfig { + /// Candidate numbers of independent environments, evaluated in ascending order. + pub environment_counts: Vec, + /// Number of paired nuisance runs planned per future environment. + pub runs_per_environment: usize, + /// Monte Carlo future-study simulations per candidate environment count. + pub simulation_trials: usize, + /// BCa resamples used across environment aggregates in each future study. + pub bootstrap_resamples: usize, + /// Desired probability of clearing the practical-effect gate. + pub target_power: f64, + /// Frozen smallest effect size of interest in the primary metric's natural units. + pub sesoi: f64, + /// Frozen alternative effect used for planning. This is deliberately separate + /// from the observed DEV mean to avoid winner's-curse sample-size selection. + pub planning_effect: f64, + /// Multiplier applied to DEV residuals around their equal-environment mean. + /// Values >= 1.0 allow conservative variance-inflation sensitivity analysis. + pub residual_scale: f64, + pub direction: PowerDirection, + pub seed: u64, +} + +impl ProspectivePowerConfig { + pub fn validate(&self) -> Result<(), String> { + if self.environment_counts.is_empty() { + return Err("at least one candidate environment count is required".into()); + } + if self.environment_counts.iter().any(|count| *count < 3) { + return Err("candidate environment counts must be at least three".into()); + } + if self.environment_counts.windows(2).any(|window| window[0] >= window[1]) { + return Err("candidate environment counts must be strictly increasing".into()); + } + if self.runs_per_environment == 0 { + return Err("runs_per_environment must be positive".into()); + } + if self.simulation_trials < 100 { + return Err("prospective power requires at least 100 simulation trials".into()); + } + if self.bootstrap_resamples < 100 { + return Err("prospective power requires at least 100 bootstrap resamples".into()); + } + if !self.target_power.is_finite() || !(0.5..=1.0).contains(&self.target_power) { + return Err("target power must be finite and between 0.5 and 1.0".into()); + } + if !self.sesoi.is_finite() || self.sesoi <= 0.0 { + return Err("SESOI must be finite and strictly positive".into()); + } + if !self.planning_effect.is_finite() { + return Err("planning effect must be finite".into()); + } + match self.direction { + PowerDirection::MeaningfulGain if self.planning_effect <= self.sesoi => { + return Err("gain planning effect must be strictly greater than +SESOI".into()); + } + PowerDirection::MeaningfulRegression if self.planning_effect >= -self.sesoi => { + return Err("regression planning effect must be strictly below -SESOI".into()); + } + _ => {} + } + if !self.residual_scale.is_finite() || self.residual_scale < 1.0 { + return Err("residual_scale must be finite and at least 1.0".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PowerPoint { + pub environments: usize, + pub successes: usize, + pub simulation_trials: usize, + pub estimated_power: f64, + pub power_ci95_low: f64, + pub power_ci95_high: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProspectivePowerPlan { + /// Digest of the exact DEV experiment manifest used for planning. + pub dev_manifest_digest: String, + /// Digest over DEV nested outcomes plus all power-planning settings. + pub planning_input_digest: String, + pub target_power: f64, + pub sesoi: f64, + pub planning_effect: f64, + pub residual_scale: f64, + pub direction: PowerDirection, + pub runs_per_environment: usize, + /// First tested count whose lower power bound clears target and remains clear + /// for every larger tested count. `None` means the tested grid was insufficient. + pub minimum_environments: Option, + pub points: Vec, +} + +/// Estimate a future CONFIRM sample size using DEV variability only. +/// +/// This is an empirical Monte Carlo planning tool, not an analytic guarantee. +/// DEV supplies the empirical environment/nuisance residual distribution, but +/// simulated effects are re-centered on the separately frozen `planning_effect` +/// rather than on the observed DEV mean. This avoids using a potentially tuned +/// or winner's-curse DEV point estimate as the alternative hypothesis. +/// +/// Each future environment is sampled from the DEV environment distribution, +/// paired nuisance runs are resampled within that environment, and residuals are +/// optionally inflated by `residual_scale`. A simulated study is successful only +/// when its environment-level BCa interval satisfies the same SESOI gate used for +/// practical-effect interpretation. +/// +/// The selected count is conservative in two ways: the lower Wilson 95% bound on +/// Monte Carlo power must clear `target_power`, and the crossing must remain clear +/// for every larger candidate count tested. Freeze the resulting plan before +/// observing CONFIRM outcomes. +pub fn prospective_power_from_dev( + dev_manifest: &ExperimentManifest, + dev_results: &[NestedEnvironmentResult], + config: &ProspectivePowerConfig, +) -> Result { + dev_manifest.validate()?; + if dev_manifest.stream_namespace != StreamNamespace::Dev + || dev_manifest.tuning_status != TuningStatus::Exploratory + { + return Err("prospective power planning must use an exploratory DEV manifest".into()); + } + validate_environment_results(dev_results)?; + config.validate()?; + if dev_results.len() != dev_manifest.seed_manifest.environment_seeds.len() { + return Err("DEV result count must match the frozen environment-seed manifest".into()); + } + + let dev_manifest_digest = dev_manifest.digest().map_err(|error| error.to_string())?; + let planning_input_digest = + digest_serialized(&(dev_manifest_digest.as_str(), dev_results, config))?; + let dev_mean = dev_results + .iter() + .map(NestedEnvironmentResult::mean_delta) + .sum::() + / dev_results.len() as f64; + + let mut rng = StdRng::seed_from_u64(config.seed); + let mut points = Vec::with_capacity(config.environment_counts.len()); + + for &environment_count in &config.environment_counts { + let mut successes = 0usize; + + for _ in 0..config.simulation_trials { + let mut future_environment_means = Vec::with_capacity(environment_count); + for _ in 0..environment_count { + let template = &dev_results[rng.gen_range(0..dev_results.len())]; + let mut delta_sum = 0.0; + for _ in 0..config.runs_per_environment { + let run = &template.paired_runs[rng.gen_range(0..template.run_count())]; + let residual = run.delta() - dev_mean; + delta_sum += config.planning_effect + config.residual_scale * residual; + } + future_environment_means.push(delta_sum / config.runs_per_environment as f64); + } + + let mean_delta = future_environment_means.iter().sum::() + / future_environment_means.len() as f64; + let ci_seed = rng.next_u64(); + let (ci95_low, ci95_high) = bootstrap_ci_bca( + &future_environment_means, + config.bootstrap_resamples, + 0.05, + ci_seed, + ); + let estimate = PairedEstimate { + n_pairs: future_environment_means.len(), + mean_delta, + ci95_low, + ci95_high, + }; + let practical = classify_practical_effect(&estimate, config.sesoi)?; + let success = matches!( + (config.direction, practical), + (PowerDirection::MeaningfulGain, PracticalEffect::MeaningfulGain) + | ( + PowerDirection::MeaningfulRegression, + PracticalEffect::MeaningfulRegression + ) + ); + if success { + successes += 1; + } + } + + let estimated_power = successes as f64 / config.simulation_trials as f64; + let (power_ci95_low, power_ci95_high) = + wilson_interval(successes, config.simulation_trials); + points.push(PowerPoint { + environments: environment_count, + successes, + simulation_trials: config.simulation_trials, + estimated_power, + power_ci95_low, + power_ci95_high, + }); + } + + let minimum_environments = points.iter().enumerate().find_map(|(index, point)| { + let sustained = points[index..] + .iter() + .all(|later| later.power_ci95_low >= config.target_power); + sustained.then_some(point.environments) + }); + + Ok(ProspectivePowerPlan { + dev_manifest_digest, + planning_input_digest, + target_power: config.target_power, + sesoi: config.sesoi, + planning_effect: config.planning_effect, + residual_scale: config.residual_scale, + direction: config.direction, + runs_per_environment: config.runs_per_environment, + minimum_environments, + points, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment::{EXPERIMENT_MANIFEST_SCHEMA_V1, SeedManifest}; + + fn digest(value: u64) -> String { + format!("{value:064x}") + } + + fn run(id: u64, delta: f64) -> PairedRunResult { + PairedRunResult { + nuisance_digest: digest(id), + candidate: 0.5 + delta, + control: 0.5, + } + } + + fn environment(id: u64, deltas: &[f64]) -> NestedEnvironmentResult { + NestedEnvironmentResult { + environment_digest: digest(10_000 + id), + paired_runs: deltas + .iter() + .enumerate() + .map(|(index, delta)| run(id * 100 + index as u64 + 1, *delta)) + .collect(), + } + } + + fn dev_manifest(environment_count: usize) -> ExperimentManifest { + ExperimentManifest { + schema: EXPERIMENT_MANIFEST_SCHEMA_V1.into(), + experiment_id: "SYM-ARCH-002A2-DEV".into(), + experiment_version: "v1".into(), + code_revision: "deadbeef".into(), + preregistration_hash: digest(1), + generator_hash: digest(2), + stream_namespace: StreamNamespace::Dev, + tuning_status: TuningStatus::Exploratory, + prior_results_observed: true, + seed_manifest: SeedManifest { + environment_seeds: (1..=environment_count as u64).collect(), + representation_seeds: vec![11, 12], + learner_seeds: vec![21, 22], + stream_seeds: vec![31, 32], + }, + primary_hypothesis: "DEV variability informs prospective power".into(), + primary_comparator: "matched control".into(), + sesoi: 0.05, + } + } + + #[test] + fn hierarchical_estimate_equal_weights_environments_not_run_counts() { + let results = vec![ + environment(1, &[0.20]), + environment(2, &[0.00; 10]), + environment(3, &[0.10, 0.10]), + ]; + let estimate = hierarchical_environment_delta_percentile(&results, 500, 42).unwrap(); + assert_eq!(estimate.n_environments, 3); + assert_eq!(estimate.total_nested_pairs, 13); + assert!((estimate.mean_delta - 0.10).abs() < 1e-12); + assert!(estimate.ci95_low.is_finite()); + assert!(estimate.ci95_high.is_finite()); + assert!(estimate.ci95_low <= estimate.ci95_high); + } + + #[test] + fn hierarchical_estimate_is_deterministic_for_fixed_seed() { + let results = vec![ + environment(1, &[0.08, 0.10, 0.12]), + environment(2, &[0.06, 0.09, 0.11]), + environment(3, &[0.07, 0.10, 0.13]), + environment(4, &[0.09, 0.11, 0.14]), + ]; + let first = hierarchical_environment_delta_percentile(&results, 500, 7).unwrap(); + let second = hierarchical_environment_delta_percentile(&results, 500, 7).unwrap(); + assert_eq!(first, second); + } + + #[test] + fn hierarchical_estimate_rejects_duplicate_environment_identity() { + let first = environment(1, &[0.1]); + let mut duplicate = environment(2, &[0.2]); + duplicate.environment_digest = first.environment_digest.clone(); + let results = vec![first, duplicate, environment(3, &[0.3])]; + assert!(hierarchical_environment_delta_percentile(&results, 500, 42).is_err()); + } + + #[test] + fn nested_environment_rejects_duplicate_nuisance_identity() { + let mut result = environment(1, &[0.1, 0.2]); + result.paired_runs[1].nuisance_digest = result.paired_runs[0].nuisance_digest.clone(); + assert!(result.validate().is_err()); + } + + #[test] + fn prospective_power_plan_is_deterministic_bound_and_finds_strong_effect() { + let dev = vec![ + environment(1, &[0.01, 0.02, 0.03]), + environment(2, &[-0.01, 0.02, 0.04]), + environment(3, &[0.02, 0.03, 0.05]), + environment(4, &[-0.02, 0.01, 0.03]), + environment(5, &[0.01, 0.03, 0.04]), + environment(6, &[0.00, 0.02, 0.03]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3, 5, 8], + runs_per_environment: 3, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.12, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 99, + }; + let manifest = dev_manifest(dev.len()); + let first = prospective_power_from_dev(&manifest, &dev, &config).unwrap(); + let second = prospective_power_from_dev(&manifest, &dev, &config).unwrap(); + assert_eq!(first, second); + assert_eq!(first.points.len(), 3); + assert!(looks_like_digest(&first.dev_manifest_digest)); + assert!(looks_like_digest(&first.planning_input_digest)); + assert!(first.minimum_environments.is_some()); + assert!(first.points.iter().all(|point| { + (0.0..=1.0).contains(&point.estimated_power) + && point.power_ci95_low <= point.estimated_power + && point.estimated_power <= point.power_ci95_high + })); + } + + #[test] + fn prospective_power_does_not_use_observed_dev_mean_as_planning_effect() { + let dev = vec![ + environment(1, &[0.30]), + environment(2, &[0.30]), + environment(3, &[0.30]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.08, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 5, + }; + let plan = prospective_power_from_dev(&dev_manifest(3), &dev, &config).unwrap(); + assert_eq!(plan.planning_effect, 0.08); + assert_eq!(plan.points[0].estimated_power, 1.0); + } + + #[test] + fn prospective_power_rejects_non_dev_manifest() { + let dev = vec![ + environment(1, &[0.1]), + environment(2, &[0.1]), + environment(3, &[0.1]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + let mut manifest = dev_manifest(3); + manifest.stream_namespace = StreamNamespace::Confirm; + manifest.tuning_status = TuningStatus::ConfirmatoryFirstUse; + manifest.prior_results_observed = false; + assert!(prospective_power_from_dev(&manifest, &dev, &config).is_err()); + } + + #[test] + fn prospective_power_rejects_manifest_result_count_mismatch() { + let dev = vec![ + environment(1, &[0.1]), + environment(2, &[0.1]), + environment(3, &[0.1]), + ]; + let config = ProspectivePowerConfig { + environment_counts: vec![3], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + assert!(prospective_power_from_dev(&dev_manifest(4), &dev, &config).is_err()); + } + + #[test] + fn prospective_power_rejects_effect_inside_practical_margin() { + let config = ProspectivePowerConfig { + environment_counts: vec![5, 8], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.04, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + assert!(config.validate().is_err()); + } + + #[test] + fn prospective_power_rejects_unsorted_environment_counts() { + let config = ProspectivePowerConfig { + environment_counts: vec![8, 5], + runs_per_environment: 1, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 1, + }; + assert!(config.validate().is_err()); + } +} diff --git a/crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs b/crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs new file mode 100644 index 000000000..942660760 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/statistics_design.rs @@ -0,0 +1,360 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Nuisance-topology guards for SYM-ARCH-002A2. +//! +//! A representation/learner/stream realization may be genuinely nested inside +//! one generated environment, or the same realization may be reused across many +//! environments. Those designs have different dependence structures and must not +//! share an uncertainty procedure silently. +//! +//! This module is the fail-closed scientific entry point around the lower-level +//! v1 statistics implementation: +//! +//! - `NestedIndependent` requires nuisance identities to be globally unique; +//! - `CrossedShared` requires the same complete nuisance grid in every environment; +//! - crossed uncertainty uses a two-way environment × nuisance bootstrap; +//! - v1 prospective power refuses crossed designs until a crossed power simulator +//! is implemented and separately validated. + +use crate::experiment::{ExperimentManifest, PairedEstimate}; +use crate::experiment_statistics::{ + NestedEnvironmentResult, ProspectivePowerConfig, ProspectivePowerPlan, + prospective_power_from_dev, +}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NuisanceTopology { + /// Every nuisance realization belongs to exactly one environment. + NestedIndependent, + /// The same nuisance identities form a complete balanced grid in every environment. + CrossedShared, +} + +fn normalized_environment_grid( + results: &[NestedEnvironmentResult], +) -> Result>, String> { + if results.len() < 3 { + return Err("at least three independent environments are required".into()); + } + + let mut environment_ids = BTreeSet::new(); + let mut grids = Vec::with_capacity(results.len()); + for environment in results { + environment.validate()?; + let environment_id = environment.environment_digest.to_ascii_lowercase(); + if !environment_ids.insert(environment_id) { + return Err("duplicate environment digest would create pseudoreplication".into()); + } + + let mut grid = BTreeMap::new(); + for run in &environment.paired_runs { + let nuisance_id = run.nuisance_digest.to_ascii_lowercase(); + if grid.insert(nuisance_id, run.delta()).is_some() { + return Err("duplicate nuisance identity within one environment".into()); + } + } + grids.push(grid); + } + Ok(grids) +} + +/// Validate the dependence topology before scientific inference. +/// +/// This is intentionally stricter than `NestedEnvironmentResult::validate`, which +/// can validate only one environment at a time and therefore cannot know whether +/// a nuisance realization has been reused across environments. +pub fn validate_nuisance_topology( + results: &[NestedEnvironmentResult], + topology: NuisanceTopology, +) -> Result<(), String> { + let grids = normalized_environment_grid(results)?; + + match topology { + NuisanceTopology::NestedIndependent => { + let mut globally_seen = BTreeSet::new(); + for grid in &grids { + for nuisance_id in grid.keys() { + if !globally_seen.insert(nuisance_id.clone()) { + return Err( + "nested-independent nuisance design reuses a nuisance identity across environments" + .into(), + ); + } + } + } + } + NuisanceTopology::CrossedShared => { + let reference: BTreeSet<&str> = grids[0].keys().map(String::as_str).collect(); + if reference.len() < 2 { + return Err( + "crossed-shared inference requires at least two nuisance identities".into(), + ); + } + for grid in grids.iter().skip(1) { + let observed: BTreeSet<&str> = grid.keys().map(String::as_str).collect(); + if observed != reference { + return Err( + "crossed-shared nuisance design requires the same complete nuisance grid in every environment" + .into(), + ); + } + } + } + } + + Ok(()) +} + +fn percentile(sorted: &[f64], probability: f64) -> f64 { + debug_assert!(!sorted.is_empty()); + let position = probability.clamp(0.0, 1.0) * (sorted.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + sorted[lower] + } else { + let weight = position - lower as f64; + sorted[lower] * (1.0 - weight) + sorted[upper] * weight + } +} + +/// Equal-cell estimate with a two-way crossed environment × nuisance bootstrap. +/// +/// Each bootstrap replicate independently resamples environment identities and +/// nuisance identities with replacement, then evaluates the Cartesian product of +/// those sampled clusters. Reusing a nuisance draw across all sampled environments +/// preserves the crossed dependence that would be destroyed by independently +/// resampling nuisance runs inside each environment. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CrossedHierarchicalEstimate { + pub n_environments: usize, + pub n_nuisance_identities: usize, + pub total_cells: usize, + pub mean_delta: f64, + pub ci95_low: f64, + pub ci95_high: f64, + pub bootstrap_resamples: usize, +} + +pub fn crossed_environment_nuisance_delta_percentile( + results: &[NestedEnvironmentResult], + n_resamples: usize, + seed: u64, +) -> Result { + validate_nuisance_topology(results, NuisanceTopology::CrossedShared)?; + if n_resamples < 200 { + return Err("crossed hierarchical bootstrap requires at least 200 resamples".into()); + } + + let grids = normalized_environment_grid(results)?; + let nuisance_ids: Vec = grids[0].keys().cloned().collect(); + let n_environments = grids.len(); + let n_nuisance_identities = nuisance_ids.len(); + let total_cells = n_environments + .checked_mul(n_nuisance_identities) + .ok_or_else(|| "crossed cell count overflow".to_string())?; + + let observed_sum: f64 = grids + .iter() + .flat_map(|grid| nuisance_ids.iter().map(move |id| grid[id])) + .sum(); + let mean_delta = observed_sum / total_cells as f64; + + let mut rng = StdRng::seed_from_u64(seed); + let mut bootstrap = Vec::with_capacity(n_resamples); + for _ in 0..n_resamples { + let sampled_environments: Vec = (0..n_environments) + .map(|_| rng.gen_range(0..n_environments)) + .collect(); + let sampled_nuisance: Vec = (0..n_nuisance_identities) + .map(|_| rng.gen_range(0..n_nuisance_identities)) + .collect(); + + let mut sum = 0.0; + for &environment_index in &sampled_environments { + let grid = &grids[environment_index]; + for &nuisance_index in &sampled_nuisance { + sum += grid[&nuisance_ids[nuisance_index]]; + } + } + bootstrap.push(sum / total_cells as f64); + } + + bootstrap.sort_by(|left, right| left.total_cmp(right)); + Ok(CrossedHierarchicalEstimate { + n_environments, + n_nuisance_identities, + total_cells, + mean_delta, + ci95_low: percentile(&bootstrap, 0.025), + ci95_high: percentile(&bootstrap, 0.975), + bootstrap_resamples: n_resamples, + }) +} + +/// Convert a crossed estimate into the common paired-estimate shape when only the +/// mean and interval are needed by downstream SESOI classification. +pub fn crossed_as_paired_estimate(estimate: &CrossedHierarchicalEstimate) -> PairedEstimate { + PairedEstimate { + n_pairs: estimate.n_environments, + mean_delta: estimate.mean_delta, + ci95_low: estimate.ci95_low, + ci95_high: estimate.ci95_high, + } +} + +/// Topology-checked entry point for v1 prospective power planning. +/// +/// The existing power simulator is valid only for genuinely nested nuisance runs. +/// Crossed designs fail closed here rather than silently receiving the nested +/// simulator. A dedicated crossed power simulator should be a separately reviewed +/// follow-up because it must reproduce both environment and shared-nuisance cluster +/// uncertainty inside each simulated study. +pub fn prospective_power_from_dev_topology_checked( + dev_manifest: &ExperimentManifest, + dev_results: &[NestedEnvironmentResult], + config: &ProspectivePowerConfig, + topology: NuisanceTopology, +) -> Result { + validate_nuisance_topology(dev_results, topology)?; + match topology { + NuisanceTopology::NestedIndependent => { + prospective_power_from_dev(dev_manifest, dev_results, config) + } + NuisanceTopology::CrossedShared => Err( + "v1 prospective power does not support crossed nuisance designs; use a separately validated crossed power simulator" + .into(), + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment::{ + EXPERIMENT_MANIFEST_SCHEMA_V1, SeedManifest, StreamNamespace, TuningStatus, + }; + use crate::experiment_statistics::{PairedRunResult, PowerDirection}; + + fn digest(value: u64) -> String { + format!("{value:064x}") + } + + fn environment(id: u64, nuisance: &[(u64, f64)]) -> NestedEnvironmentResult { + NestedEnvironmentResult { + environment_digest: digest(10_000 + id), + paired_runs: nuisance + .iter() + .map(|(nuisance_id, delta)| PairedRunResult { + nuisance_digest: digest(*nuisance_id), + candidate: 0.5 + delta, + control: 0.5, + }) + .collect(), + } + } + + #[test] + fn nested_topology_rejects_cross_environment_nuisance_reuse() { + let results = vec![ + environment(1, &[(1, 0.1)]), + environment(2, &[(1, 0.2)]), + environment(3, &[(3, 0.3)]), + ]; + assert!( + validate_nuisance_topology(&results, NuisanceTopology::NestedIndependent).is_err() + ); + } + + #[test] + fn crossed_topology_requires_a_complete_shared_grid() { + let good = vec![ + environment(1, &[(1, 0.10), (2, 0.20)]), + environment(2, &[(1, 0.00), (2, 0.10)]), + environment(3, &[(1, -0.10), (2, 0.00)]), + ]; + validate_nuisance_topology(&good, NuisanceTopology::CrossedShared).unwrap(); + + let bad = vec![ + environment(1, &[(1, 0.10), (2, 0.20)]), + environment(2, &[(1, 0.00), (3, 0.10)]), + environment(3, &[(1, -0.10), (2, 0.00)]), + ]; + assert!(validate_nuisance_topology(&bad, NuisanceTopology::CrossedShared).is_err()); + } + + #[test] + fn crossed_bootstrap_is_deterministic_and_equal_cell_weighted() { + let results = vec![ + environment(1, &[(1, 0.10), (2, 0.20), (3, 0.30)]), + environment(2, &[(1, 0.00), (2, 0.10), (3, 0.20)]), + environment(3, &[(1, -0.10), (2, 0.00), (3, 0.10)]), + environment(4, &[(1, 0.20), (2, 0.30), (3, 0.40)]), + ]; + let first = crossed_environment_nuisance_delta_percentile(&results, 500, 42).unwrap(); + let second = crossed_environment_nuisance_delta_percentile(&results, 500, 42).unwrap(); + assert_eq!(first, second); + assert_eq!(first.n_environments, 4); + assert_eq!(first.n_nuisance_identities, 3); + assert_eq!(first.total_cells, 12); + assert!((first.mean_delta - 0.15).abs() < 1e-12); + assert!(first.ci95_low <= first.mean_delta); + assert!(first.mean_delta <= first.ci95_high); + } + + #[test] + fn crossed_design_fails_closed_in_v1_power_planner() { + let results = vec![ + environment(1, &[(1, 0.08), (2, 0.12)]), + environment(2, &[(1, 0.06), (2, 0.10)]), + environment(3, &[(1, 0.07), (2, 0.11)]), + ]; + let manifest = ExperimentManifest { + schema: EXPERIMENT_MANIFEST_SCHEMA_V1.into(), + experiment_id: "crossed-power-test".into(), + experiment_version: "v1".into(), + code_revision: "deadbeef".into(), + preregistration_hash: digest(100), + generator_hash: digest(101), + stream_namespace: StreamNamespace::Dev, + tuning_status: TuningStatus::Exploratory, + prior_results_observed: true, + seed_manifest: SeedManifest { + environment_seeds: vec![1, 2, 3], + representation_seeds: vec![1, 2], + learner_seeds: vec![], + stream_seeds: vec![], + }, + primary_hypothesis: "crossed power is fail-closed in v1".into(), + primary_comparator: "matched control".into(), + sesoi: 0.05, + }; + let config = ProspectivePowerConfig { + environment_counts: vec![3, 5], + runs_per_environment: 2, + simulation_trials: 100, + bootstrap_resamples: 100, + target_power: 0.80, + sesoi: 0.05, + planning_effect: 0.10, + residual_scale: 1.0, + direction: PowerDirection::MeaningfulGain, + seed: 7, + }; + + let error = prospective_power_from_dev_topology_checked( + &manifest, + &results, + &config, + NuisanceTopology::CrossedShared, + ) + .unwrap_err(); + assert!(error.contains("does not support crossed nuisance designs")); + } +} diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index df0a8b5ea..0e12062ca 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -56,6 +56,10 @@ pub mod benchmarks; pub mod experiment; #[path = "experiment/confirmatory.rs"] pub mod experiment_confirmatory; +#[path = "experiment/statistics.rs"] +pub mod experiment_statistics; +#[path = "experiment/statistics_design.rs"] +pub mod experiment_statistics_design; pub mod harness; pub mod substrate_transfer; pub mod wm; @@ -64,4 +68,4 @@ pub mod wm; pub mod neuroevolution_fitness; #[cfg(test)] -mod proptest_math_benchmarks; +mod proptest_math_benchmarks; \ No newline at end of file diff --git a/docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md b/docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md new file mode 100644 index 000000000..6fde33b7a --- /dev/null +++ b/docs/research/SYM_ARCH_002A2_HIERARCHICAL_POWER_V1.md @@ -0,0 +1,143 @@ +# SYM-ARCH-002A2 Hierarchical Statistics and Prospective Power v1 + +**Status:** statistical-infrastructure tranche +**Scientific claim status:** none +**Parent plan:** issue #55 +**Stacked dependency:** PR #57 (`research/sym-arch-002a-core-v1`) + +## Purpose + +SYM-ARCH-002A2 extends the experimental core with two pieces needed before a future CONFIRM run can be sized or interpreted responsibly: + +1. hierarchical candidate-minus-control uncertainty with **generated environments as the independent unit**; +2. prospective sample-size planning from DEV variability without using the observed DEV mean as the assumed confirmatory effect. + +This tranche does not choose a CONFIRM sample size because no qualifying DEV campaign is part of this PR. It supplies the instrument that will make that choice later from a frozen DEV artifact and a separately frozen planning effect. + +## Hierarchical result schema + +`NestedEnvironmentResult` contains: + +- one unique environment digest; +- one or more `PairedRunResult` entries; +- a unique nuisance-run digest for each nested representation/learner/stream realization; +- paired candidate/control outcomes from that same nuisance realization. + +Validation rejects: + +- malformed environment or nuisance digests; +- duplicate environment identities; +- duplicate nuisance-run identities within an environment; +- empty nested-run sets; +- non-finite outcomes. + +Nested runs reduce within-environment uncertainty. They do **not** increase the number of independent environments. + +## Hierarchical bootstrap + +`hierarchical_environment_delta_percentile` performs a two-level paired bootstrap: + +1. sample environments with replacement; +2. inside each sampled environment, sample paired nuisance runs with replacement; +3. compute one mean delta for each sampled environment; +4. average environment means with **equal environment weight**; +5. repeat and report a deterministic 95% percentile interval. + +An environment with ten nuisance runs therefore does not receive ten times the scientific weight of an environment with one nuisance run. + +v1 deliberately calls this a **hierarchical percentile interval**, not BCa. The existing one-level BCa implementation is not silently relabeled as a nested-data method. + +## Prospective power planning + +`prospective_power_from_dev` is allowed only with a valid exploratory `DEV` manifest. The number of DEV environment results must match the frozen environment-seed manifest. + +The planner uses DEV for empirical heterogeneity/noise, but **not for the assumed effect size**. This prevents a tuned or unusually favorable DEV point estimate from becoming a winner's-curse input to CONFIRM sizing. + +The preregistered planning configuration includes: + +- candidate independent-environment counts; +- nested paired runs planned per future environment; +- Monte Carlo trial count; +- BCa resamples across future environment aggregates; +- target power; +- SESOI; +- a separately frozen `planning_effect`; +- a residual-variance scale; +- gain/regression direction; +- RNG seed. + +For a gain claim, `planning_effect` must be strictly greater than `+SESOI`. For a regression-detection plan it must be strictly below `-SESOI`. + +### DEV winner-bias guard + +Let `d_dev` be the equal-environment DEV mean delta. A sampled DEV run contributes a residual: + +`residual = run_delta - d_dev` + +The simulated future delta is: + +`planning_effect + residual_scale * residual` + +Thus DEV contributes the observed environment/nuisance variability while the future-study center remains the independently frozen planning effect. + +`residual_scale >= 1` supports conservative sensitivity analysis. A value below one is rejected because it would make the observed DEV variability artificially easier. + +## Practical-effect-aligned power criterion + +A simulated future study counts as successful only when its environment-level 95% BCa interval clears the same SESOI practical-effect gate used by 002A1. + +Power is therefore not defined as merely obtaining `p < 0.05`. It is the probability that the future study will support the **predeclared practically meaningful claim**. + +For each candidate environment count, A2 reports: + +- Monte Carlo success count; +- point estimate of power; +- Wilson 95% interval for that estimated power. + +The selected `minimum_environments`, if any, is the first tested count whose **lower Wilson power bound** clears the target **and remains above target for every larger tested count**. This prevents a single noisy threshold crossing from setting CONFIRM size. + +If no tested count satisfies this sustained conservative rule, `minimum_environments = None`; the correct response is to extend the planning grid or reconsider the design, not to weaken the target after looking at results. + +## Provenance binding + +The power plan records: + +- digest of the exact DEV experiment manifest; +- digest over the DEV nested outcomes and complete power configuration. + +Changing DEV data, planning effect, SESOI, variance inflation, candidate counts, simulation settings, or seed therefore changes the planning-input digest. + +The chosen CONFIRM sample size should later be frozen by referencing this digest before CONFIRM outcomes are observed. + +## Acceptance gate + +The exact stacked PR head should pass: + +1. rustfmt on A2 Rust paths; +2. focused `experiment_statistics` tests; +3. `cargo check -p symthaea-psych-bench --lib`; +4. equal-environment-weight reference test; +5. deterministic hierarchical-bootstrap test; +6. duplicate environment/nuisance identity rejection; +7. DEV-only planning rejection for non-DEV manifests; +8. manifest/result environment-count binding; +9. planning-effect-outside-SESOI validation; +10. deterministic power-plan/provenance-digest test; +11. Wilson-bounded power reporting and sustained-threshold selection. + +## Explicit non-claims and limitations + +A2 v1 does **not** claim: + +- that percentile hierarchical intervals are optimal for every future metric; +- that the empirical DEV variance distribution perfectly represents CONFIRM; +- that a planning effect should be chosen from DEV performance; +- that any particular number of environments is currently required; +- that 002 has demonstrated an architecture advantage; +- that multiple-comparison correction, resource Pareto analysis, or procedural benchmark validity is complete. + +The `planning_effect` must be justified and frozen outside this code path. Sensitivity analysis should run multiple preregistered `residual_scale >= 1` conditions where DEV is small or heterogeneous. + +## Merge boundary + +This PR is statistical infrastructure and may merge on correctness after its exact-head gate passes. It should remain stacked on #57 until #57 merges; after that, rebase/retarget to `main` without changing the statistical contract. diff --git a/docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md b/docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md new file mode 100644 index 000000000..873e690fb --- /dev/null +++ b/docs/research/SYM_ARCH_002A2_NUISANCE_TOPOLOGY_V1.md @@ -0,0 +1,82 @@ +# SYM-ARCH-002A2 — Nuisance Topology Boundary v1 + +**Status:** statistical design hardening; no architecture result + +**Tracks:** #55, stacked in #58 + +## Why this exists + +A representation/learner/stream realization can enter an experiment in two materially different ways: + +1. **nested-independent** — each nuisance realization belongs to only one generated environment; +2. **crossed-shared** — the same nuisance realization is reused across multiple generated environments. + +Those observations do not have the same dependence structure. + +If the same representation seed, learner seed, stream ordering, or other nuisance realization is reused across environments, independently resampling nuisance runs inside each environment can treat a shared source of variation as though it were independent. That can distort uncertainty and later power calculations. + +The topology is therefore part of the frozen statistical design, not an implementation detail. + +## `NestedIndependent` + +The scientific guard requires every nuisance digest to be globally unique across environments. + +The existing environment-first + nested-run hierarchical percentile bootstrap is appropriate only after this condition is established. + +A repeated nuisance digest across two environments is rejected instead of being silently treated as nested. + +## `CrossedShared` + +A crossed design must expose the same complete nuisance-identity grid in every environment. Missing or substituted nuisance cells make the design invalid for the v1 crossed estimator. + +A2 now supplies a two-way environment × nuisance percentile bootstrap: + +1. sample environments with replacement; +2. sample nuisance identities with replacement **once per replicate**; +3. apply those sampled nuisance identities across all sampled environments; +4. average the sampled Cartesian product; +5. report the 2.5% and 97.5% bootstrap percentiles. + +Sampling the nuisance identities jointly across environments preserves the fact that a reused nuisance realization is shared rather than independent within each environment. + +The v1 crossed estimator treats the combined representation/learner/stream nuisance digest as one crossed cluster. If later experiments need separate variance components for representation seed, learner seed, and stream seed, that is a distinct multiway-random-effects extension and must not be inferred from this combined digest. + +## Prospective power boundary + +The existing v1 prospective-power simulator was written for nested-independent nuisance runs. + +The topology-checked scientific entry point therefore behaves as follows: + +- `NestedIndependent` → validate global uniqueness, then use the existing DEV-based prospective-power simulator; +- `CrossedShared` → **fail closed** with an explicit unsupported-design error. + +A crossed CONFIRM design must not use the nested power recommendation. + +Before crossed prospective power is allowed, a later implementation must simulate both independent environment sampling and shared nuisance-cluster sampling inside each future study and validate that procedure separately. + +This restriction is intentional. Missing power support is safer than a precise-looking sample-size recommendation from the wrong dependence model. + +## Confirmatory freeze requirements + +Before opening CONFIRM, freeze at least: + +- nuisance topology (`nested_independent` or `crossed_shared`); +- exact definition of the nuisance digest; +- whether representation, learner, and stream seeds are reused across environments; +- environment count; +- nuisance count or runs per environment; +- missing-cell policy (v1 crossed requires none); +- uncertainty procedure; +- prospective-power procedure and version; +- bootstrap seeds/resample counts; +- primary metric, comparator, SESOI, and direction. + +Changing from nested to crossed after observing results creates a new analysis specification; it is not a harmless re-labeling. + +## Claim ceiling + +This hardening supports only: + +> SYM-ARCH-002A2 distinguishes nested-independent from balanced crossed nuisance designs, applies topology-specific uncertainty checks, and refuses to use the v1 nested prospective-power simulator for crossed data. + +It does not establish that either design is preferable, does not choose the eventual CONFIRM topology, and carries no Symthaea capability claim. \ No newline at end of file