From 03c315b2f6a93d7468e93be730d93458a70e5f53 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:51:21 +0200 Subject: [PATCH 1/7] feat(research): add construct-validity shortcut controls --- .../src/experiment/construct_validity.rs | 846 ++++++++++++++++++ 1 file changed, 846 insertions(+) create mode 100644 crates/domains/symthaea-psych-bench/src/experiment/construct_validity.rs diff --git a/crates/domains/symthaea-psych-bench/src/experiment/construct_validity.rs b/crates/domains/symthaea-psych-bench/src/experiment/construct_validity.rs new file mode 100644 index 000000000..280056a73 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/construct_validity.rs @@ -0,0 +1,846 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Adversarial construct-validity controls for generated architecture benchmarks. +//! +//! A benchmark that is structurally valid can still be scientifically weak if a +//! trivial train-only shortcut solves its evaluation split. This module attacks +//! generated tasks with deliberately simple controls before any architecture +//! result is interpreted. +//! +//! The controls consume learner-visible `features` only. Example ids and +//! `support_tags` are metadata and are never supplied to fitted shortcut models. + +use crate::experiment::TaskProgram; +use crate::experiment_validity::{ + evaluate_rule, validate_generated_task, BenchmarkValidityPolicy, BenchmarkValidityReport, + ExampleRecord, GeneratedTaskDataset, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +pub const CONSTRUCT_VALIDITY_SCHEMA_V1: &str = "symthaea.construct-validity/v1"; + +const CHANCE_HASH_DOMAIN: &[u8] = b"symthaea.construct-validity.chance/v1"; +const SHUFFLE_HASH_DOMAIN: &[u8] = b"symthaea.construct-validity.shuffle/v1"; + +fn accuracy(predictions: &[bool], examples: &[ExampleRecord]) -> Result { + if predictions.len() != examples.len() || examples.is_empty() { + return Err("predictions must match a non-empty evaluation split".into()); + } + let correct = predictions + .iter() + .zip(examples) + .filter(|(prediction, example)| **prediction == example.expected_label) + .count(); + Ok(correct as f64 / examples.len() as f64) +} + +fn majority_label(labels: &[bool]) -> Result { + if labels.is_empty() { + return Err("majority predictor requires training labels".into()); + } + let positives = labels.iter().filter(|label| **label).count(); + // Deterministic negative-class tie break. The choice is arbitrary but frozen. + Ok(positives * 2 > labels.len()) +} + +fn labels(examples: &[ExampleRecord]) -> Vec { + examples.iter().map(|example| example.expected_label).collect() +} + +fn learner_feature_keys(examples: &[ExampleRecord]) -> BTreeSet { + examples + .iter() + .flat_map(|example| example.features.keys().cloned()) + .collect() +} + +fn feature_distance(left: &ExampleRecord, right: &ExampleRecord) -> usize { + let keys: BTreeSet<&String> = left + .features + .keys() + .chain(right.features.keys()) + .collect(); + keys.into_iter() + .filter(|key| left.features.get(*key) != right.features.get(*key)) + .count() +} + +#[derive(Debug, Clone)] +struct SingleFeatureModel { + feature: String, + value_labels: BTreeMap, + fallback: bool, + train_accuracy: f64, +} + +fn fit_single_feature_model( + train: &[ExampleRecord], + train_labels: &[bool], +) -> Result { + if train.is_empty() || train.len() != train_labels.len() { + return Err("single-feature model requires paired non-empty training data".into()); + } + let fallback = majority_label(train_labels)?; + let keys = learner_feature_keys(train); + if keys.is_empty() { + return Err("single-feature model requires learner-visible features".into()); + } + + let mut best: Option = None; + for feature in keys { + let mut counts: BTreeMap = BTreeMap::new(); + for (example, label) in train.iter().zip(train_labels) { + if let Some(value) = example.features.get(&feature) { + let entry = counts.entry(*value).or_default(); + if *label { + entry.0 += 1; + } else { + entry.1 += 1; + } + } + } + let value_labels: BTreeMap = counts + .into_iter() + .map(|(value, (positive, negative))| { + let label = if positive == negative { + fallback + } else { + positive > negative + }; + (value, label) + }) + .collect(); + let predictions: Vec = train + .iter() + .map(|example| { + example + .features + .get(&feature) + .and_then(|value| value_labels.get(value)) + .copied() + .unwrap_or(fallback) + }) + .collect(); + let correct = predictions + .iter() + .zip(train_labels) + .filter(|(prediction, label)| **prediction == **label) + .count(); + let train_accuracy = correct as f64 / train.len() as f64; + + let candidate = SingleFeatureModel { + feature, + value_labels, + fallback, + train_accuracy, + }; + let replace = match &best { + None => true, + Some(current) => { + candidate.train_accuracy > current.train_accuracy + || (candidate.train_accuracy == current.train_accuracy + && candidate.feature < current.feature) + } + }; + if replace { + best = Some(candidate); + } + } + + best.ok_or_else(|| "failed to fit single-feature model".into()) +} + +fn predict_single_feature(model: &SingleFeatureModel, example: &ExampleRecord) -> bool { + example + .features + .get(&model.feature) + .and_then(|value| model.value_labels.get(value)) + .copied() + .unwrap_or(model.fallback) +} + +fn exact_lookup_predictions( + train: &[ExampleRecord], + train_labels: &[bool], + eval: &[ExampleRecord], +) -> Result, String> { + if train.is_empty() || train.len() != train_labels.len() { + return Err("exact lookup requires paired non-empty training data".into()); + } + let fallback = majority_label(train_labels)?; + let mut counts: BTreeMap = BTreeMap::new(); + for (example, label) in train.iter().zip(train_labels) { + let digest = example.feature_digest()?; + let entry = counts.entry(digest).or_default(); + if *label { + entry.0 += 1; + } else { + entry.1 += 1; + } + } + Ok(eval + .iter() + .map(|example| { + example + .feature_digest() + .ok() + .and_then(|digest| counts.get(&digest).copied()) + .map(|(positive, negative)| { + if positive == negative { + fallback + } else { + positive > negative + } + }) + .unwrap_or(fallback) + }) + .collect()) +} + +fn nearest_neighbor_predictions( + train: &[ExampleRecord], + train_labels: &[bool], + eval: &[ExampleRecord], +) -> Result, String> { + if train.is_empty() || train.len() != train_labels.len() { + return Err("nearest neighbor requires paired non-empty training data".into()); + } + let fallback = majority_label(train_labels)?; + let mut predictions = Vec::with_capacity(eval.len()); + for target in eval { + let mut best_distance = usize::MAX; + let mut positive = 0usize; + let mut negative = 0usize; + for (candidate, label) in train.iter().zip(train_labels) { + let distance = feature_distance(candidate, target); + if distance < best_distance { + best_distance = distance; + positive = 0; + negative = 0; + } + if distance == best_distance { + if *label { + positive += 1; + } else { + negative += 1; + } + } + } + predictions.push(if positive == negative { + fallback + } else { + positive > negative + }); + } + Ok(predictions) +} + +fn deterministic_chance_prediction(seed: u64, example_id: &str) -> bool { + let mut hasher = blake3::Hasher::new(); + hasher.update(CHANCE_HASH_DOMAIN); + hasher.update(&[0]); + hasher.update(&seed.to_le_bytes()); + hasher.update(&[0]); + hasher.update(example_id.as_bytes()); + hasher.finalize().as_bytes()[0] & 1 == 1 +} + +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut value = *state; + value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + value ^ (value >> 31) +} + +fn shuffle_seed(seed: u64) -> u64 { + let mut hasher = blake3::Hasher::new(); + hasher.update(SHUFFLE_HASH_DOMAIN); + hasher.update(&[0]); + hasher.update(&seed.to_le_bytes()); + let bytes = hasher.finalize(); + u64::from_le_bytes(bytes.as_bytes()[..8].try_into().expect("eight bytes")) +} + +fn shuffled_labels(original: &[bool], seed: u64) -> Result, String> { + if original.len() < 2 { + return Err("label shuffle requires at least two training examples".into()); + } + let mut shuffled = original.to_vec(); + let mut state = shuffle_seed(seed); + for index in (1..shuffled.len()).rev() { + let swap = (splitmix64(&mut state) % (index as u64 + 1)) as usize; + shuffled.swap(index, swap); + } + Ok(shuffled) +} + +/// 95% Wilson upper bound for a fair Bernoulli classifier's realized accuracy. +/// This is used only as a benchmark-resolution check, not as a hypothesis test. +fn chance_accuracy_upper95(n: usize) -> Result { + if n == 0 { + return Err("chance interval requires a non-empty evaluation split".into()); + } + let p = 0.5; + let z = 1.959_963_984_540_054_f64; + let z2 = z * z; + let n = n as f64; + let denominator = 1.0 + z2 / n; + let center = (p + z2 / (2.0 * n)) / denominator; + let half = z * ((p * (1.0 - p) / n + z2 / (4.0 * n * n)).sqrt()) / denominator; + Ok((center + half).min(1.0)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ShortcutControlKind { + Majority, + SingleFeatureMarginal, + ExactLookup, + NearestNeighbor, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ShortcutControlScore { + pub kind: ShortcutControlKind, + pub train_accuracy: Option, + pub eval_accuracy: f64, + pub selected_feature: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ShuffledRelationSummary { + pub seeds: Vec, + pub single_feature_mean_accuracy: f64, + pub single_feature_max_accuracy: f64, + pub nearest_neighbor_mean_accuracy: f64, + pub nearest_neighbor_max_accuracy: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConstructValidityPolicy { + /// Structural/oracle policy inherited from SYM-ARCH-002A3. + pub structural_policy: BenchmarkValidityPolicy, + /// Maximum allowed evaluation accuracy for any train-only shortcut. Reaching + /// this ceiling is sufficient to make the benchmark inconclusive. + pub shortcut_accuracy_ceiling: f64, + /// Maximum allowed mean evaluation accuracy across shuffled-label controls. + pub shuffled_mean_accuracy_ceiling: f64, + /// Minimum required executable-oracle accuracy. + pub oracle_accuracy_floor: f64, + /// Deterministic seed for the realized chance sanity control. + pub chance_seed: u64, + /// Frozen seeds for prevalence-preserving shuffled-label controls. + pub shuffle_seeds: Vec, +} + +impl ConstructValidityPolicy { + pub fn validate(&self) -> Result<(), String> { + for (name, value) in [ + ("shortcut_accuracy_ceiling", self.shortcut_accuracy_ceiling), + ( + "shuffled_mean_accuracy_ceiling", + self.shuffled_mean_accuracy_ceiling, + ), + ("oracle_accuracy_floor", self.oracle_accuracy_floor), + ] { + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + return Err(format!("{name} must be a finite probability")); + } + } + if self.shortcut_accuracy_ceiling <= 0.5 { + return Err("shortcut ceiling must be above fair-chance accuracy".into()); + } + if self.shuffled_mean_accuracy_ceiling <= 0.5 { + return Err("shuffled-label ceiling must be above fair-chance accuracy".into()); + } + if self.oracle_accuracy_floor < 0.5 { + return Err("oracle accuracy floor must be at least chance".into()); + } + if self.shuffle_seeds.len() < 4 { + return Err("at least four frozen shuffle seeds are required".into()); + } + let unique: BTreeSet = self.shuffle_seeds.iter().copied().collect(); + if unique.len() != self.shuffle_seeds.len() { + return Err("shuffle seeds must be unique".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ConstructValidityStatus { + Passed, + InconclusiveBenchmark, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ConstructViolationKind { + StructuralValidityFailed, + InsufficientEvaluationResolution, + OracleControlFailed, + ShortcutControlTooStrong, + ShuffledRelationControlTooStrong, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ConstructViolation { + pub kind: ConstructViolationKind, + pub detail: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConstructValidityReport { + pub schema: String, + pub status: ConstructValidityStatus, + pub dataset_digest: Option, + pub structural_report: BenchmarkValidityReport, + pub eval_examples: usize, + pub chance_expected_accuracy: f64, + pub chance_observed_accuracy: Option, + pub chance_accuracy_upper95: Option, + pub oracle_accuracy: Option, + pub shortcut_scores: Vec, + pub shuffled_relation: Option, + pub violations: Vec, +} + +impl ConstructValidityReport { + pub fn passed(&self) -> bool { + self.status == ConstructValidityStatus::Passed + } +} + +fn push_violation( + violations: &mut Vec, + kind: ConstructViolationKind, + detail: impl Into, +) { + violations.push(ConstructViolation { + kind, + detail: detail.into(), + }); +} + +fn mean(values: &[f64]) -> f64 { + values.iter().sum::() / values.len() as f64 +} + +/// Run adversarial construct-validity controls against a generated task. +/// +/// Model selection for the single-feature control uses training accuracy only. +/// Evaluation labels are used solely for final scoring. Exact lookup and nearest +/// neighbor receive only training features/labels plus evaluation features. +/// Support tags and example ids are excluded from fitted shortcut inputs. +pub fn run_construct_validity( + program: &TaskProgram, + dataset: &GeneratedTaskDataset, + policy: &ConstructValidityPolicy, +) -> Result { + policy.validate()?; + let structural_report = validate_generated_task(program, dataset, &policy.structural_policy); + let mut violations = Vec::new(); + + if !structural_report.is_valid() { + push_violation( + &mut violations, + ConstructViolationKind::StructuralValidityFailed, + "SYM-ARCH-002A3 structural/oracle validity must pass before shortcut controls count", + ); + return Ok(ConstructValidityReport { + schema: CONSTRUCT_VALIDITY_SCHEMA_V1.to_string(), + status: ConstructValidityStatus::InconclusiveBenchmark, + dataset_digest: structural_report.dataset_digest.clone(), + structural_report, + eval_examples: dataset.eval.len(), + chance_expected_accuracy: 0.5, + chance_observed_accuracy: None, + chance_accuracy_upper95: None, + oracle_accuracy: None, + shortcut_scores: Vec::new(), + shuffled_relation: None, + violations, + }); + } + + let dataset_digest = Some(dataset.digest()?); + let eval_examples = dataset.eval.len(); + let chance_upper = chance_accuracy_upper95(eval_examples)?; + let resolution_ceiling = policy + .shortcut_accuracy_ceiling + .min(policy.shuffled_mean_accuracy_ceiling); + if chance_upper >= resolution_ceiling { + push_violation( + &mut violations, + ConstructViolationKind::InsufficientEvaluationResolution, + format!( + "fair-chance 95% upper accuracy {chance_upper:.4} reaches/exceeds the tighter preregistered shortcut ceiling {resolution_ceiling:.4}; increase evaluation support" + ), + ); + } + + let chance_predictions: Vec = dataset + .eval + .iter() + .map(|example| deterministic_chance_prediction(policy.chance_seed, &example.example_id)) + .collect(); + let chance_observed_accuracy = accuracy(&chance_predictions, &dataset.eval)?; + + let oracle_predictions: Vec = dataset + .eval + .iter() + .map(|example| evaluate_rule(&program.rule, &example.features)) + .collect::, _>>()?; + let oracle_accuracy = accuracy(&oracle_predictions, &dataset.eval)?; + if oracle_accuracy < policy.oracle_accuracy_floor { + push_violation( + &mut violations, + ConstructViolationKind::OracleControlFailed, + format!( + "symbolic oracle accuracy {oracle_accuracy:.4} is below floor {:.4}", + policy.oracle_accuracy_floor + ), + ); + } + + let train_labels = labels(&dataset.train); + let train_majority = majority_label(&train_labels)?; + let majority_predictions = vec![train_majority; dataset.eval.len()]; + let majority_train_accuracy = train_labels + .iter() + .filter(|label| **label == train_majority) + .count() as f64 + / train_labels.len() as f64; + let majority_score = ShortcutControlScore { + kind: ShortcutControlKind::Majority, + train_accuracy: Some(majority_train_accuracy), + eval_accuracy: accuracy(&majority_predictions, &dataset.eval)?, + selected_feature: None, + }; + + let single_feature_model = fit_single_feature_model(&dataset.train, &train_labels)?; + let single_feature_predictions: Vec = dataset + .eval + .iter() + .map(|example| predict_single_feature(&single_feature_model, example)) + .collect(); + let single_feature_score = ShortcutControlScore { + kind: ShortcutControlKind::SingleFeatureMarginal, + train_accuracy: Some(single_feature_model.train_accuracy), + eval_accuracy: accuracy(&single_feature_predictions, &dataset.eval)?, + selected_feature: Some(single_feature_model.feature.clone()), + }; + + let exact_lookup_predictions = + exact_lookup_predictions(&dataset.train, &train_labels, &dataset.eval)?; + let exact_lookup_score = ShortcutControlScore { + kind: ShortcutControlKind::ExactLookup, + train_accuracy: Some(1.0), + eval_accuracy: accuracy(&exact_lookup_predictions, &dataset.eval)?, + selected_feature: None, + }; + + let nearest_predictions = + nearest_neighbor_predictions(&dataset.train, &train_labels, &dataset.eval)?; + let nearest_score = ShortcutControlScore { + kind: ShortcutControlKind::NearestNeighbor, + train_accuracy: Some(1.0), + eval_accuracy: accuracy(&nearest_predictions, &dataset.eval)?, + selected_feature: None, + }; + + let shortcut_scores = vec![ + majority_score, + single_feature_score, + exact_lookup_score, + nearest_score, + ]; + for score in &shortcut_scores { + if score.eval_accuracy >= policy.shortcut_accuracy_ceiling { + push_violation( + &mut violations, + ConstructViolationKind::ShortcutControlTooStrong, + format!( + "{:?} evaluation accuracy {:.4} reaches/exceeds ceiling {:.4}", + score.kind, score.eval_accuracy, policy.shortcut_accuracy_ceiling + ), + ); + } + } + + let mut shuffled_single_feature = Vec::with_capacity(policy.shuffle_seeds.len()); + let mut shuffled_nearest = Vec::with_capacity(policy.shuffle_seeds.len()); + for &seed in &policy.shuffle_seeds { + let permuted = shuffled_labels(&train_labels, seed)?; + let model = fit_single_feature_model(&dataset.train, &permuted)?; + let predictions: Vec = dataset + .eval + .iter() + .map(|example| predict_single_feature(&model, example)) + .collect(); + shuffled_single_feature.push(accuracy(&predictions, &dataset.eval)?); + + let predictions = nearest_neighbor_predictions(&dataset.train, &permuted, &dataset.eval)?; + shuffled_nearest.push(accuracy(&predictions, &dataset.eval)?); + } + let single_feature_mean_accuracy = mean(&shuffled_single_feature); + let nearest_neighbor_mean_accuracy = mean(&shuffled_nearest); + let shuffled_relation = ShuffledRelationSummary { + seeds: policy.shuffle_seeds.clone(), + single_feature_mean_accuracy, + single_feature_max_accuracy: shuffled_single_feature + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max), + nearest_neighbor_mean_accuracy, + nearest_neighbor_max_accuracy: shuffled_nearest + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max), + }; + + for (name, value) in [ + ("single_feature", single_feature_mean_accuracy), + ("nearest_neighbor", nearest_neighbor_mean_accuracy), + ] { + if value >= policy.shuffled_mean_accuracy_ceiling { + push_violation( + &mut violations, + ConstructViolationKind::ShuffledRelationControlTooStrong, + format!( + "shuffled-label {name} mean accuracy {value:.4} reaches/exceeds ceiling {:.4}", + policy.shuffled_mean_accuracy_ceiling + ), + ); + } + } + + Ok(ConstructValidityReport { + schema: CONSTRUCT_VALIDITY_SCHEMA_V1.to_string(), + status: if violations.is_empty() { + ConstructValidityStatus::Passed + } else { + ConstructValidityStatus::InconclusiveBenchmark + }, + dataset_digest, + structural_report, + eval_examples, + chance_expected_accuracy: 0.5, + chance_observed_accuracy: Some(chance_observed_accuracy), + chance_accuracy_upper95: Some(chance_upper), + oracle_accuracy: Some(oracle_accuracy), + shortcut_scores, + shuffled_relation: Some(shuffled_relation), + violations, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment::{ + ContextVisibility, RuleExpr, TaskProgram, TimingRegime, TASK_PROGRAM_SCHEMA_V1, + }; + use crate::experiment_validity::symbolic_oracle_digest; + + fn xor_rule() -> RuleExpr { + RuleExpr::Xor { + left: Box::new(RuleExpr::Parity { + factor: "a".into(), + modulus: 2, + remainder: 0, + }), + right: Box::new(RuleExpr::Parity { + factor: "b".into(), + modulus: 2, + remainder: 0, + }), + } + } + + fn example(id: &str, a: i64, b: i64, split: &str, rule: &RuleExpr) -> ExampleRecord { + let features = BTreeMap::from([("a".to_string(), a), ("b".to_string(), b)]); + let expected_label = evaluate_rule(rule, &features).unwrap(); + ExampleRecord { + example_id: id.into(), + features, + support_tags: vec![split.into()], + expected_label, + } + } + + fn clean_dataset(eval_count: usize) -> (TaskProgram, GeneratedTaskDataset) { + let rule = xor_rule(); + let train_pairs = [ + (0, 0), + (0, 1), + (1, 0), + (1, 1), + (2, 2), + (2, 3), + (3, 2), + (3, 3), + ]; + let eval_pairs = [ + (0, 2), + (0, 3), + (1, 2), + (1, 3), + (2, 0), + (2, 1), + (3, 0), + (3, 1), + ]; + let train: Vec<_> = train_pairs + .into_iter() + .enumerate() + .map(|(index, (a, b))| example(&format!("train-{index}"), a, b, "train", &rule)) + .collect(); + let eval: Vec<_> = eval_pairs + .into_iter() + .take(eval_count) + .enumerate() + .map(|(index, (a, b))| example(&format!("eval-{index}"), a, b, "eval", &rule)) + .collect(); + let positive_examples = train + .iter() + .chain(&eval) + .filter(|example| example.expected_label) + .count(); + let negative_examples = train.len() + eval.len() - positive_examples; + let mut program = TaskProgram { + schema: TASK_PROGRAM_SCHEMA_V1.into(), + program_id: "construct-validity-xor".into(), + family: "unit-test".into(), + rule, + context_visibility: ContextVisibility::TaskFree, + timing_regime: TimingRegime::Uniform, + positive_examples, + negative_examples, + train_support: vec!["train".into()], + eval_support: vec!["eval".into()], + oracle_digest: String::new(), + }; + program.oracle_digest = symbolic_oracle_digest(&program.rule).unwrap(); + let dataset = GeneratedTaskDataset { + program_digest: program.digest().unwrap(), + train, + eval, + }; + (program, dataset) + } + + fn policy() -> ConstructValidityPolicy { + ConstructValidityPolicy { + structural_policy: BenchmarkValidityPolicy::task_free_strict(), + shortcut_accuracy_ceiling: 0.80, + shuffled_mean_accuracy_ceiling: 0.80, + oracle_accuracy_floor: 1.0, + chance_seed: 17, + shuffle_seeds: vec![1, 2, 3, 4, 5, 7, 9, 11], + } + } + + #[test] + fn clean_xor_benchmark_passes_shortcut_gate() { + let (program, dataset) = clean_dataset(8); + let report = run_construct_validity(&program, &dataset, &policy()).unwrap(); + assert!(report.passed(), "violations: {:?}", report.violations); + assert_eq!(report.oracle_accuracy, Some(1.0)); + assert_eq!(report.eval_examples, 8); + assert!(report.chance_accuracy_upper95.unwrap() < 0.80); + assert!(report + .shortcut_scores + .iter() + .all(|score| score.eval_accuracy < 0.80)); + } + + #[test] + fn single_feature_label_channel_is_rejected() { + let (program, mut dataset) = clean_dataset(8); + for example in dataset.train.iter_mut().chain(&mut dataset.eval) { + example.features.insert( + "shortcut".into(), + if example.expected_label { 1 } else { 0 }, + ); + } + let report = run_construct_validity(&program, &dataset, &policy()).unwrap(); + assert_eq!(report.status, ConstructValidityStatus::InconclusiveBenchmark); + assert!(report.violations.iter().any(|violation| { + violation.kind == ConstructViolationKind::ShortcutControlTooStrong + })); + let marginal = report + .shortcut_scores + .iter() + .find(|score| score.kind == ShortcutControlKind::SingleFeatureMarginal) + .unwrap(); + assert_eq!(marginal.selected_feature.as_deref(), Some("shortcut")); + assert_eq!(marginal.eval_accuracy, 1.0); + } + + #[test] + fn four_item_eval_is_too_coarse_for_eighty_percent_ceiling() { + let (program, dataset) = clean_dataset(4); + let report = run_construct_validity(&program, &dataset, &policy()).unwrap(); + assert_eq!(report.status, ConstructValidityStatus::InconclusiveBenchmark); + assert!(report.violations.iter().any(|violation| { + violation.kind == ConstructViolationKind::InsufficientEvaluationResolution + })); + assert!(report.chance_accuracy_upper95.unwrap() >= 0.80); + } + + #[test] + fn support_metadata_never_enters_shortcut_models() { + let (mut program, dataset) = clean_dataset(8); + let baseline = run_construct_validity(&program, &dataset, &policy()).unwrap(); + + program.train_support = vec!["train".into(), "positive".into(), "negative".into()]; + program.eval_support = vec!["eval".into(), "positive".into(), "negative".into()]; + let mut tagged = dataset.clone(); + for example in tagged.train.iter_mut().chain(&mut tagged.eval) { + example.support_tags.push(if example.expected_label { + "positive".into() + } else { + "negative".into() + }); + } + tagged.program_digest = program.digest().unwrap(); + let retagged = run_construct_validity(&program, &tagged, &policy()).unwrap(); + + assert_eq!(baseline.shortcut_scores, retagged.shortcut_scores); + assert_eq!(baseline.oracle_accuracy, retagged.oracle_accuracy); + } + + #[test] + fn shuffled_labels_preserve_prevalence() { + let (_, dataset) = clean_dataset(8); + let original = labels(&dataset.train); + let positives = original.iter().filter(|label| **label).count(); + for seed in [1, 2, 3, 4, 5] { + let shuffled = shuffled_labels(&original, seed).unwrap(); + assert_eq!( + shuffled.iter().filter(|label| **label).count(), + positives + ); + } + } + + #[test] + fn policy_rejects_weak_or_duplicate_shuffle_contracts() { + let mut invalid = policy(); + invalid.shuffle_seeds = vec![1, 2, 3]; + assert!(invalid.validate().is_err()); + + invalid = policy(); + invalid.shuffle_seeds = vec![1, 2, 3, 3]; + assert!(invalid.validate().is_err()); + + invalid = policy(); + invalid.shortcut_accuracy_ceiling = 0.5; + assert!(invalid.validate().is_err()); + } +} From 9f304ed25bc72497e6741719cfa79841a687bdef Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:51:43 +0200 Subject: [PATCH 2/7] feat(research): expose construct-validity controls --- crates/domains/symthaea-psych-bench/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index 634ef01f5..188dd26c2 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -56,6 +56,8 @@ pub mod benchmarks; pub mod experiment; #[path = "experiment/confirmatory.rs"] pub mod experiment_confirmatory; +#[path = "experiment/construct_validity.rs"] +pub mod experiment_construct_validity; #[path = "experiment/validity.rs"] pub mod experiment_validity; pub mod harness; From 7ce5ad36e5c51ed4895dd2e6daae469c9339e96f Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:55:57 +0200 Subject: [PATCH 3/7] fix(test): remove stale loop trial fixture field --- .../symthaea-psych-bench/src/harness/neuromod_correlation.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs index b4c849fdc..c6056aad3 100644 --- a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs +++ b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs @@ -241,11 +241,6 @@ mod tests { cycle_time_us: 1000 + (i as u64 * 50), learning_occurred: i % 5 == 0, reward: 0.0, - // Added 2026-07-31 to unblock the crate: another session's - // LoopTrialResult gained `cycle_reward` and this test fixture was - // collateral damage. Synthetic data, so 0.0 matches the - // neighbouring `reward`/`moral_score` fixtures. - cycle_reward: 0.0, oxytocin: 0.3, moral_score: 0.0, bath_entropy: 0.5, From d90a3ad2fb9e9d523c7e6bf910141ef9b967f983 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:56:55 +0200 Subject: [PATCH 4/7] docs(research): freeze 002A4 construct-validity contract --- .../SYM_ARCH_002A4_CONSTRUCT_VALIDITY_V1.md | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 docs/research/SYM_ARCH_002A4_CONSTRUCT_VALIDITY_V1.md diff --git a/docs/research/SYM_ARCH_002A4_CONSTRUCT_VALIDITY_V1.md b/docs/research/SYM_ARCH_002A4_CONSTRUCT_VALIDITY_V1.md new file mode 100644 index 000000000..827912243 --- /dev/null +++ b/docs/research/SYM_ARCH_002A4_CONSTRUCT_VALIDITY_V1.md @@ -0,0 +1,198 @@ +# SYM-ARCH-002A4 — Adversarial Construct Validity v1 + +## Purpose + +SYM-ARCH-002A3 checks that a generated benchmark is internally coherent: its executable oracle agrees with labels, provenance is bound, train/evaluation examples do not leak structurally, task identity is not exposed under a strict task-free policy, and known corruptions are rejected. + +002A4 asks the next question: + +> **Can a scientifically trivial train-only shortcut solve the benchmark anyway?** + +If yes, the benchmark is not allowed to support an architecture claim, even if the candidate architecture performs well. + +This tranche is measurement infrastructure only. It produces no Symthaea capability claim. + +## Dependency + +002A4 is stacked on SYM-ARCH-002A3 / PR #59 because it consumes `TaskProgram`, `GeneratedTaskDataset`, the executable symbolic oracle, and the A3 structural validity report. + +A3 must pass before A4 scores count. + +## Learner-visible boundary + +Shortcut models receive only `ExampleRecord.features` and training labels. + +They do **not** receive: + +- `example_id` as a fitted predictor input; +- `support_tags`; +- TaskProgram support metadata; +- oracle outputs; +- evaluation labels during fitting/model selection. + +The deterministic chance sanity control hashes `example_id` only to generate a reproducible fair-coin prediction. That id is not exposed to fitted shortcut models. + +A unit test changes support tags to make them perfectly label-correlated and verifies that fitted shortcut scores do not change. + +## Positive and negative controls + +### 1. Executable symbolic oracle — positive control + +The A3 `RuleExpr` interpreter predicts the evaluation labels. + +The preregistration freezes `oracle_accuracy_floor`. For a deterministic synthetic task, this should normally be `1.0`. + +If the oracle misses the floor, the benchmark is `INCONCLUSIVE_BENCHMARK`; architecture scores are not interpreted. + +### 2. Deterministic fair-chance sanity control + +A domain-separated BLAKE3 hash over a frozen `chance_seed` and evaluation example id generates a deterministic 50/50 prediction. + +The observed chance accuracy is diagnostic only. One realized chance draw is **not** itself a pass/fail hypothesis test. + +### 3. Training-majority predictor + +The predictor chooses the majority label from the training split only, with a frozen negative-class tie break, then applies it to evaluation examples. + +This exposes evaluation imbalance or a task that can be solved by ignoring all input features. + +### 4. Single-feature marginal predictor + +For each learner-visible feature separately: + +1. fit a value-to-majority-label table on training data; +2. score that feature on training data; +3. select the feature with highest **training** accuracy only; +4. break feature-selection ties lexicographically; +5. evaluate the selected model on the held-out split; +6. fall back to the training-majority label for unseen feature values. + +Evaluation labels are not used to select the feature. + +This attacks label channels, marginal correlations, and high-cardinality single-feature shortcuts. + +### 5. Exact lookup + +The control hashes the complete learner-visible feature assignment for each training example and memorizes its label. Unseen evaluation feature assignments fall back to the training-majority label. + +Under A3's strict feature-disjoint policy, exact lookup should not obtain an advantage. It remains an explicit sanity control so a relaxed future split policy cannot silently turn memorization into “generalization.” + +### 6. Nearest neighbor + +A deterministic categorical nearest-neighbor control uses Hamming-style distance over the union of learner-visible feature keys. Missing or unequal values count as mismatches. + +All training examples at the minimum distance vote; ties fall back to the training-majority label. + +This attacks local interpolation and near-duplicate structure that exact lookup does not catch. + +### 7. Shuffled-relation negative controls + +Training labels are deterministically permuted while preserving class prevalence. For every frozen shuffle seed, A4 refits: + +- the single-feature marginal predictor; +- nearest neighbor. + +It reports both mean and maximum evaluation accuracy across shuffles. The **mean** is the v1 gate to avoid making the verdict depend on one lucky permutation; maxima remain visible diagnostics. + +At least four unique shuffle seeds are required. Confirmatory use should freeze a larger seed set when compute is cheap. + +## Finite-sample resolution gate + +A shortcut threshold is meaningless if the evaluation split is so small that ordinary chance fluctuation can reach it. + +A4 therefore computes the 95% Wilson upper bound for a fair Bernoulli classifier at the actual evaluation-set size. + +Let: + +- `C_shortcut` = preregistered train-only shortcut accuracy ceiling; +- `C_shuffle` = preregistered shuffled-label mean accuracy ceiling; +- `U_chance95` = fair-chance 95% Wilson upper accuracy bound. + +The benchmark is `INCONCLUSIVE_BENCHMARK` when: + +`U_chance95 >= min(C_shortcut, C_shuffle)`. + +This is a benchmark-resolution check, not a null-hypothesis significance test. + +One immediate consequence is intentional: very small held-outs such as the four-item SYM-ARCH-001 compositional split cannot support a fine-grained shortcut ceiling. A harder follow-up benchmark must increase evaluation support rather than infer precision from a few Bernoulli outcomes. + +## Frozen policy inputs + +Before a benchmark becomes claim-bearing, freeze all of the following outside the result path: + +- `shortcut_accuracy_ceiling`; +- `shuffled_mean_accuracy_ceiling`; +- `oracle_accuracy_floor`; +- `chance_seed`; +- the complete unique `shuffle_seeds` set; +- A3 structural validity policy; +- benchmark generator/version and seed manifest. + +Do not weaken a ceiling, swap shuffle seeds, enlarge evaluation support selectively, or remove a shortcut control after observing architecture or construct-validity results under the same experiment version. + +The `0.80` ceilings used in unit tests are synthetic test fixtures only. They are **not** recommended scientific defaults and are not preregistered thresholds for SYM-ARCH-002. + +## Fail-closed verdict + +A4 has only two top-level states: + +- `PASSED` +- `INCONCLUSIVE_BENCHMARK` + +It does not return `NEGATIVE` for an architecture when a benchmark fails. + +`INCONCLUSIVE_BENCHMARK` is produced if any of the following occurs: + +1. A3 structural/oracle validity fails; +2. finite evaluation support cannot resolve the frozen ceiling; +3. executable oracle accuracy is below its frozen floor; +4. majority, single-feature, exact-lookup, or nearest-neighbor evaluation accuracy reaches/exceeds `shortcut_accuracy_ceiling`; +5. the mean shuffled-label single-feature or nearest-neighbor accuracy reaches/exceeds `shuffled_mean_accuracy_ceiling`. + +A benchmark failure means **fix or redesign the instrument before interpreting architecture performance**. + +## What A4 does not establish + +Passing A4 v1 does not prove the absence of every possible shortcut. It only rules out the specific low-complexity alternatives implemented here under the frozen policy. + +Later construct-validity work may add, where justified: + +- two-feature interaction controls; +- regularized logistic/linear controls; +- decision-tree controls; +- support/serialization leakage audits at the runtime boundary; +- temporal/order-only predictors; +- relation-grammar equivalence checks; +- counterfactual feature interventions; +- learned representation probes. + +Those additions must be named as stronger controls rather than retroactively changing the meaning of an A4 v1 pass. + +## Acceptance criteria + +The exact PR head must demonstrate: + +1. A3 structural validity is a hard prerequisite; +2. oracle positive control is explicit; +3. majority/chance controls are reported; +4. single-feature selection uses training data only; +5. exact lookup cannot silently use metadata; +6. nearest-neighbor distance is deterministic and feature-only; +7. shuffled labels preserve training prevalence; +8. at least four unique shuffle seeds are required; +9. tiny evaluation splits fail the finite-resolution gate; +10. an injected single-feature label channel makes the benchmark inconclusive; +11. label-correlated `support_tags` do not change shortcut scores; +12. no architecture score or capability claim is produced by this tranche. + +## Wording ceiling + +A passing result supports only: + +> **The benchmark passed the implemented v1 structural/oracle and low-complexity shortcut controls under the frozen policy.** + +It does not support: + +> **The benchmark is shortcut-free.** + +and it does not by itself support any claim that Symthaea is superior to a baseline architecture. From 84babd91d6aade95a537d40f6ae56a248528773c Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:57:11 +0200 Subject: [PATCH 5/7] ci(research): gate 002A4 construct-validity controls --- .../sym-arch-002a4-construct-validity.yml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/sym-arch-002a4-construct-validity.yml diff --git a/.github/workflows/sym-arch-002a4-construct-validity.yml b/.github/workflows/sym-arch-002a4-construct-validity.yml new file mode 100644 index 000000000..6b105c35b --- /dev/null +++ b/.github/workflows/sym-arch-002a4-construct-validity.yml @@ -0,0 +1,65 @@ +name: SYM-ARCH-002A4 Construct Validity + +on: + pull_request: + branches: + - main + - research/sym-arch-002a3-validity-v1 + paths: + - '.github/workflows/sym-arch-002a4-construct-validity.yml' + - 'crates/domains/symthaea-psych-bench/src/experiment/construct_validity.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/validity.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/mod.rs' + - 'crates/domains/symthaea-psych-bench/src/lib.rs' + - 'crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' + - 'docs/research/SYM_ARCH_002A4_CONSTRUCT_VALIDITY_V1.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + construct-validity: + name: Validate adversarial shortcut controls + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + with: + components: rustfmt + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-002a4-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-002a4- + + - name: Check A4 Rust formatting + run: | + rustfmt --edition 2024 --check \ + crates/domains/symthaea-psych-bench/src/experiment/construct_validity.rs \ + crates/domains/symthaea-psych-bench/src/lib.rs + + - name: Run construct-validity tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_construct_validity:: -- --nocapture + + - name: Check psych-bench library + run: | + cargo check -p symthaea-psych-bench --lib From 332188b534c48addf80afdeff9a3f52c718c7350 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 20:10:12 +0200 Subject: [PATCH 6/7] ci(research): supersede stale A4 validation runs --- .github/workflows/sym-arch-002a4-construct-validity.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/sym-arch-002a4-construct-validity.yml b/.github/workflows/sym-arch-002a4-construct-validity.yml index 6b105c35b..6fba4d4e9 100644 --- a/.github/workflows/sym-arch-002a4-construct-validity.yml +++ b/.github/workflows/sym-arch-002a4-construct-validity.yml @@ -18,6 +18,12 @@ on: - 'Cargo.lock' workflow_dispatch: +concurrency: + # Automatic branch/PR validation keeps only the newest run. Manual validation + # is deliberate and receives a unique group through github.run_id. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + permissions: contents: read From 445ac8adcc63c5db557bed83c4f4f27a9bacec93 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Wed, 26 Aug 2026 09:29:23 +0200 Subject: [PATCH 7/7] ci(research): defer A4 dedicated gate while draft --- .github/workflows/sym-arch-002a4-construct-validity.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sym-arch-002a4-construct-validity.yml b/.github/workflows/sym-arch-002a4-construct-validity.yml index 6fba4d4e9..846a32584 100644 --- a/.github/workflows/sym-arch-002a4-construct-validity.yml +++ b/.github/workflows/sym-arch-002a4-construct-validity.yml @@ -2,6 +2,7 @@ name: SYM-ARCH-002A4 Construct Validity on: pull_request: + types: [opened, synchronize, reopened, ready_for_review] branches: - main - research/sym-arch-002a3-validity-v1 @@ -34,6 +35,9 @@ env: jobs: construct-validity: name: Validate adversarial shortcut controls + # Draft stacked PRs cannot merge and should not consume scarce runner capacity. + # Promoting to ready-for-review explicitly triggers this exact gate. + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest timeout-minutes: 20