From ea417172872774d4d200d02c6f17a2c69e026754 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:15:11 +0200 Subject: [PATCH 1/8] feat(research): add symbolic oracle and benchmark validity mutations --- .../src/experiment/validity.rs | 702 ++++++++++++++++++ 1 file changed, 702 insertions(+) create mode 100644 crates/domains/symthaea-psych-bench/src/experiment/validity.rs diff --git a/crates/domains/symthaea-psych-bench/src/experiment/validity.rs b/crates/domains/symthaea-psych-bench/src/experiment/validity.rs new file mode 100644 index 000000000..12f7428b6 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/validity.rs @@ -0,0 +1,702 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Fail-closed benchmark validity and mutation testing for architecture research. +//! +//! This module validates generated task datasets before any architecture score is +//! interpreted. It provides an executable symbolic oracle for the v1 `RuleExpr` +//! language, split/leakage checks, support-contract checks, deterministic dataset +//! identity, and adversarial benchmark mutations that the validator must detect. + +use crate::experiment::{RuleExpr, TaskProgram}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +const RULE_ORACLE_HASH_DOMAIN: &[u8] = b"symthaea.rule-oracle.hash/v1"; +const EXAMPLE_FEATURE_HASH_DOMAIN: &[u8] = b"symthaea.example-features.hash/v1"; +const DATASET_HASH_DOMAIN: &[u8] = b"symthaea.generated-task-dataset.hash/v1"; + +fn canonical_hash(domain: &[u8], value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?; + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(&[0]); + hasher.update(&bytes); + Ok(hasher.finalize().to_hex().to_string()) +} + +fn looks_like_digest(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +/// Deterministic digest of the executable symbolic oracle represented by a rule. +pub fn symbolic_oracle_digest(rule: &RuleExpr) -> Result { + canonical_hash(RULE_ORACLE_HASH_DOMAIN, rule) +} + +/// Evaluate the v1 symbolic rule over an integer-valued feature assignment. +/// +/// `Eq` and `Ne` interpret both operands as feature names. `Parity` reads one +/// named integer factor. Missing factors fail closed rather than defaulting. +pub fn evaluate_rule(rule: &RuleExpr, features: &BTreeMap) -> Result { + let value = |name: &str| { + features + .get(name) + .copied() + .ok_or_else(|| format!("missing feature required by oracle: {name}")) + }; + + match rule { + RuleExpr::Eq { left, right } => Ok(value(left)? == value(right)?), + RuleExpr::Ne { left, right } => Ok(value(left)? != value(right)?), + RuleExpr::Parity { + factor, + modulus, + remainder, + } => { + if *modulus == 0 || *remainder >= *modulus { + return Err("invalid parity rule reached oracle execution".into()); + } + let modulus = i64::from(*modulus); + let remainder = i64::from(*remainder); + Ok(value(factor)?.rem_euclid(modulus) == remainder) + } + RuleExpr::Not { inner } => Ok(!evaluate_rule(inner, features)?), + RuleExpr::And { terms } => { + for term in terms { + if !evaluate_rule(term, features)? { + return Ok(false); + } + } + Ok(true) + } + RuleExpr::Or { terms } => { + for term in terms { + if evaluate_rule(term, features)? { + return Ok(true); + } + } + Ok(false) + } + RuleExpr::Xor { left, right } => { + Ok(evaluate_rule(left, features)? ^ evaluate_rule(right, features)?) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExampleRecord { + pub example_id: String, + pub features: BTreeMap, + pub support_tags: Vec, + pub expected_label: bool, +} + +impl ExampleRecord { + pub fn validate_structure(&self) -> Result<(), String> { + if self.example_id.trim().is_empty() { + return Err("example id must be non-empty".into()); + } + if self.features.is_empty() { + return Err("example must contain at least one feature".into()); + } + let mut tags = BTreeSet::new(); + for tag in &self.support_tags { + let normalized = tag.trim(); + if normalized.is_empty() { + return Err("support tag must be non-empty".into()); + } + if !tags.insert(normalized.to_string()) { + return Err("duplicate support tag on example".into()); + } + } + Ok(()) + } + + /// Feature-only identity used to detect train/evaluation leakage independent + /// of labels, ids, or support annotations. + pub fn feature_digest(&self) -> Result { + canonical_hash(EXAMPLE_FEATURE_HASH_DOMAIN, &self.features) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GeneratedTaskDataset { + pub program_digest: String, + pub train: Vec, + pub eval: Vec, +} + +impl GeneratedTaskDataset { + /// Canonical set-style digest. Example ordering does not change dataset + /// identity, but ids/features/labels/support tags do. + pub fn digest(&self) -> Result { + fn canonical_examples(examples: &[ExampleRecord]) -> Vec { + let mut canonical = examples.to_vec(); + for example in &mut canonical { + example.support_tags.sort(); + } + canonical.sort_by(|left, right| left.example_id.cmp(&right.example_id)); + canonical + } + + canonical_hash( + DATASET_HASH_DOMAIN, + &( + self.program_digest.as_str(), + canonical_examples(&self.train), + canonical_examples(&self.eval), + ), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BenchmarkValidityPolicy { + /// Feature keys forbidden because they reveal task/world/boundary identity. + pub forbidden_feature_keys: BTreeSet, + /// Reject feature-identical examples across train and evaluation splits. + pub require_feature_disjoint_splits: bool, + /// Require every support tag to be declared by the split's TaskProgram support. + pub require_declared_support_tags: bool, + /// Require positive and negative examples in both train and evaluation splits. + pub require_both_classes_per_split: bool, +} + +impl BenchmarkValidityPolicy { + /// Conservative default for latent-context/task-free confirmatory tasks. + pub fn task_free_strict() -> Self { + Self { + forbidden_feature_keys: [ + "__task_id", + "task_id", + "__world_id", + "world_id", + "boundary_marker", + "time_to_switch", + ] + .into_iter() + .map(str::to_string) + .collect(), + require_feature_disjoint_splits: true, + require_declared_support_tags: true, + require_both_classes_per_split: true, + } + } + + /// Explicit-context experiments may construct a narrower forbidden-key set, + /// but doing so is an explicit policy decision rather than an implicit leak. + pub fn with_forbidden_feature_keys(mut self, keys: impl IntoIterator) -> Self { + self.forbidden_feature_keys = keys.into_iter().collect(); + self + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValidityViolationKind { + ProgramInvalid, + ProgramDigestMismatch, + OracleDigestMismatch, + DatasetEmpty, + ExampleInvalid, + DuplicateExampleId, + TrainEvalFeatureLeak, + ForbiddenFeatureLeak, + UndeclaredSupport, + OracleMismatch, + DeclaredClassCountMismatch, + SplitClassDegeneracy, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidityViolation { + pub kind: ValidityViolationKind, + pub detail: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BenchmarkValidityStatus { + Valid, + Invalid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BenchmarkValidityReport { + pub status: BenchmarkValidityStatus, + pub dataset_digest: Option, + pub train_examples: usize, + pub eval_examples: usize, + pub positives: usize, + pub negatives: usize, + pub violations: Vec, +} + +impl BenchmarkValidityReport { + pub fn is_valid(&self) -> bool { + self.status == BenchmarkValidityStatus::Valid + } +} + +fn push_violation( + violations: &mut Vec, + kind: ValidityViolationKind, + detail: impl Into, +) { + violations.push(ValidityViolation { + kind, + detail: detail.into(), + }); +} + +/// Validate a generated benchmark before any architecture result can be emitted. +pub fn validate_generated_task( + program: &TaskProgram, + dataset: &GeneratedTaskDataset, + policy: &BenchmarkValidityPolicy, +) -> BenchmarkValidityReport { + let mut violations = Vec::new(); + + if let Err(error) = program.validate() { + push_violation( + &mut violations, + ValidityViolationKind::ProgramInvalid, + error, + ); + } + + let expected_program_digest = program.digest().map_err(|error| error.to_string()); + match expected_program_digest { + Ok(digest) if dataset.program_digest != digest => push_violation( + &mut violations, + ValidityViolationKind::ProgramDigestMismatch, + "dataset is not bound to the supplied TaskProgram", + ), + Err(error) => push_violation( + &mut violations, + ValidityViolationKind::ProgramInvalid, + format!("program digest failed: {error}"), + ), + _ => {} + } + + match symbolic_oracle_digest(&program.rule) { + Ok(digest) if program.oracle_digest.to_ascii_lowercase() != digest => push_violation( + &mut violations, + ValidityViolationKind::OracleDigestMismatch, + "TaskProgram oracle digest does not match its executable RuleExpr oracle", + ), + Err(error) => push_violation( + &mut violations, + ValidityViolationKind::ProgramInvalid, + format!("symbolic oracle digest failed: {error}"), + ), + _ => {} + } + + if dataset.train.is_empty() || dataset.eval.is_empty() { + push_violation( + &mut violations, + ValidityViolationKind::DatasetEmpty, + "training and evaluation splits must both be non-empty", + ); + } + + let train_support: BTreeSet = program + .train_support + .iter() + .map(|tag| tag.trim().to_string()) + .collect(); + let eval_support: BTreeSet = program + .eval_support + .iter() + .map(|tag| tag.trim().to_string()) + .collect(); + let forbidden: BTreeSet = policy + .forbidden_feature_keys + .iter() + .map(|key| key.trim().to_ascii_lowercase()) + .collect(); + + let mut ids = BTreeSet::new(); + let mut train_feature_digests = BTreeSet::new(); + let mut positives = 0usize; + let mut negatives = 0usize; + let mut train_positive = 0usize; + let mut train_negative = 0usize; + let mut eval_positive = 0usize; + let mut eval_negative = 0usize; + + for (split_name, examples, declared_support) in [ + ("train", &dataset.train, &train_support), + ("eval", &dataset.eval, &eval_support), + ] { + for example in examples { + if let Err(error) = example.validate_structure() { + push_violation( + &mut violations, + ValidityViolationKind::ExampleInvalid, + format!("{}:{}: {error}", split_name, example.example_id), + ); + } + + if !ids.insert(example.example_id.clone()) { + push_violation( + &mut violations, + ValidityViolationKind::DuplicateExampleId, + format!("duplicate example id: {}", example.example_id), + ); + } + + for key in example.features.keys() { + if forbidden.contains(&key.to_ascii_lowercase()) { + push_violation( + &mut violations, + ValidityViolationKind::ForbiddenFeatureLeak, + format!("{}:{} contains forbidden feature {key}", split_name, example.example_id), + ); + } + } + + if policy.require_declared_support_tags { + for tag in &example.support_tags { + let normalized = tag.trim().to_string(); + if !declared_support.contains(&normalized) { + push_violation( + &mut violations, + ValidityViolationKind::UndeclaredSupport, + format!( + "{}:{} uses undeclared support tag {}", + split_name, example.example_id, normalized + ), + ); + } + } + } + + match evaluate_rule(&program.rule, &example.features) { + Ok(label) if label != example.expected_label => push_violation( + &mut violations, + ValidityViolationKind::OracleMismatch, + format!("{}:{} label disagrees with symbolic oracle", split_name, example.example_id), + ), + Err(error) => push_violation( + &mut violations, + ValidityViolationKind::OracleMismatch, + format!("{}:{} oracle failed: {error}", split_name, example.example_id), + ), + _ => {} + } + + if example.expected_label { + positives += 1; + if split_name == "train" { + train_positive += 1; + } else { + eval_positive += 1; + } + } else { + negatives += 1; + if split_name == "train" { + train_negative += 1; + } else { + eval_negative += 1; + } + } + + if split_name == "train" { + if let Ok(digest) = example.feature_digest() { + train_feature_digests.insert(digest); + } + } + } + } + + if policy.require_feature_disjoint_splits { + for example in &dataset.eval { + if let Ok(digest) = example.feature_digest() { + if train_feature_digests.contains(&digest) { + push_violation( + &mut violations, + ValidityViolationKind::TrainEvalFeatureLeak, + format!("evaluation example {} duplicates training features", example.example_id), + ); + } + } + } + } + + if positives != program.positive_examples || negatives != program.negative_examples { + push_violation( + &mut violations, + ValidityViolationKind::DeclaredClassCountMismatch, + format!( + "observed class counts +{} / -{} do not match declared +{} / -{}", + positives, negatives, program.positive_examples, program.negative_examples + ), + ); + } + + if policy.require_both_classes_per_split + && (train_positive == 0 || train_negative == 0 || eval_positive == 0 || eval_negative == 0) + { + push_violation( + &mut violations, + ValidityViolationKind::SplitClassDegeneracy, + "both train and evaluation splits must contain both classes", + ); + } + + let dataset_digest = dataset.digest().ok(); + BenchmarkValidityReport { + status: if violations.is_empty() { + BenchmarkValidityStatus::Valid + } else { + BenchmarkValidityStatus::Invalid + }, + dataset_digest, + train_examples: dataset.train.len(), + eval_examples: dataset.eval.len(), + positives, + negatives, + violations, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BenchmarkMutation { + FlipFirstEvalLabel, + LeakFirstTrainExampleIntoEval, + CorruptProgramDigest, + InjectForbiddenTaskId, + InjectUndeclaredEvalSupport, +} + +/// Apply an intentionally invalid mutation used to test the validator itself. +pub fn apply_mutation( + dataset: &GeneratedTaskDataset, + mutation: BenchmarkMutation, +) -> Result { + let mut mutated = dataset.clone(); + match mutation { + BenchmarkMutation::FlipFirstEvalLabel => { + let example = mutated + .eval + .first_mut() + .ok_or_else(|| "mutation requires a non-empty evaluation split".to_string())?; + example.expected_label = !example.expected_label; + } + BenchmarkMutation::LeakFirstTrainExampleIntoEval => { + let train = mutated + .train + .first() + .cloned() + .ok_or_else(|| "mutation requires a non-empty training split".to_string())?; + let eval = mutated + .eval + .first_mut() + .ok_or_else(|| "mutation requires a non-empty evaluation split".to_string())?; + eval.features = train.features; + } + BenchmarkMutation::CorruptProgramDigest => { + mutated.program_digest = "00".repeat(32); + } + BenchmarkMutation::InjectForbiddenTaskId => { + let example = mutated + .eval + .first_mut() + .ok_or_else(|| "mutation requires a non-empty evaluation split".to_string())?; + example.features.insert("__task_id".into(), 1); + } + BenchmarkMutation::InjectUndeclaredEvalSupport => { + let example = mutated + .eval + .first_mut() + .ok_or_else(|| "mutation requires a non-empty evaluation split".to_string())?; + example.support_tags.push("undeclared-support".into()); + } + } + Ok(mutated) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MutationDetection { + pub mutation: BenchmarkMutation, + pub detected: bool, + pub violation_kinds: Vec, +} + +/// Run the standard adversarial benchmark mutations and require each to become invalid. +pub fn mutation_detection_suite( + program: &TaskProgram, + dataset: &GeneratedTaskDataset, + policy: &BenchmarkValidityPolicy, +) -> Result, String> { + let baseline = validate_generated_task(program, dataset, policy); + if !baseline.is_valid() { + return Err("mutation suite requires a valid baseline dataset".into()); + } + + let mutations = [ + BenchmarkMutation::FlipFirstEvalLabel, + BenchmarkMutation::LeakFirstTrainExampleIntoEval, + BenchmarkMutation::CorruptProgramDigest, + BenchmarkMutation::InjectForbiddenTaskId, + BenchmarkMutation::InjectUndeclaredEvalSupport, + ]; + + let mut detections = Vec::with_capacity(mutations.len()); + for mutation in mutations { + let mutated = apply_mutation(dataset, mutation)?; + let report = validate_generated_task(program, &mutated, policy); + detections.push(MutationDetection { + mutation, + detected: !report.is_valid(), + violation_kinds: report.violations.into_iter().map(|violation| violation.kind).collect(), + }); + } + Ok(detections) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::experiment::{ + ContextVisibility, RuleExpr, TASK_PROGRAM_SCHEMA_V1, TimingRegime, + }; + + fn example(id: &str, x: i64, support: &str) -> ExampleRecord { + let mut features = BTreeMap::new(); + features.insert("x".into(), x); + ExampleRecord { + example_id: id.into(), + features, + support_tags: vec![support.into()], + expected_label: x.rem_euclid(2) == 0, + } + } + + fn fixture() -> (TaskProgram, GeneratedTaskDataset, BenchmarkValidityPolicy) { + let rule = RuleExpr::Parity { + factor: "x".into(), + modulus: 2, + remainder: 0, + }; + let oracle_digest = symbolic_oracle_digest(&rule).unwrap(); + let program = TaskProgram { + schema: TASK_PROGRAM_SCHEMA_V1.into(), + program_id: "parity-validity-001".into(), + family: "relational".into(), + rule, + context_visibility: ContextVisibility::TaskFree, + timing_regime: TimingRegime::Uniform, + positive_examples: 4, + negative_examples: 4, + train_support: vec!["train-range".into()], + eval_support: vec!["eval-range".into()], + oracle_digest, + }; + let program_digest = program.digest().unwrap(); + let dataset = GeneratedTaskDataset { + program_digest, + train: vec![ + example("train-0", 0, "train-range"), + example("train-1", 1, "train-range"), + example("train-2", 2, "train-range"), + example("train-3", 3, "train-range"), + ], + eval: vec![ + example("eval-4", 4, "eval-range"), + example("eval-5", 5, "eval-range"), + example("eval-6", 6, "eval-range"), + example("eval-7", 7, "eval-range"), + ], + }; + (program, dataset, BenchmarkValidityPolicy::task_free_strict()) + } + + #[test] + fn symbolic_oracle_handles_nested_boolean_rules() { + let rule = RuleExpr::And { + terms: vec![ + RuleExpr::Parity { + factor: "x".into(), + modulus: 2, + remainder: 0, + }, + RuleExpr::Ne { + left: "x".into(), + right: "y".into(), + }, + ], + }; + let mut features = BTreeMap::new(); + features.insert("x".into(), 4); + features.insert("y".into(), 3); + assert!(evaluate_rule(&rule, &features).unwrap()); + features.insert("y".into(), 4); + assert!(!evaluate_rule(&rule, &features).unwrap()); + } + + #[test] + fn valid_fixture_passes_and_digest_ignores_example_order() { + let (program, dataset, policy) = fixture(); + let report = validate_generated_task(&program, &dataset, &policy); + assert!(report.is_valid(), "violations={:?}", report.violations); + + let mut reordered = dataset.clone(); + reordered.train.reverse(); + reordered.eval.reverse(); + assert_eq!(dataset.digest().unwrap(), reordered.digest().unwrap()); + } + + #[test] + fn oracle_digest_must_match_executable_rule() { + let (mut program, dataset, policy) = fixture(); + program.oracle_digest = "ab".repeat(32); + let report = validate_generated_task(&program, &dataset, &policy); + assert!(report.violations.iter().any(|violation| { + violation.kind == ValidityViolationKind::OracleDigestMismatch + })); + } + + #[test] + fn feature_identical_train_eval_example_is_leakage_even_with_new_id() { + let (program, mut dataset, policy) = fixture(); + dataset.eval[0].features = dataset.train[0].features.clone(); + let report = validate_generated_task(&program, &dataset, &policy); + assert!(report.violations.iter().any(|violation| { + violation.kind == ValidityViolationKind::TrainEvalFeatureLeak + })); + } + + #[test] + fn mutation_suite_detects_every_standard_benchmark_corruption() { + let (program, dataset, policy) = fixture(); + let detections = mutation_detection_suite(&program, &dataset, &policy).unwrap(); + assert_eq!(detections.len(), 5); + assert!(detections.iter().all(|detection| detection.detected)); + } + + #[test] + fn forbidden_task_identity_feature_fails_closed() { + let (program, mut dataset, policy) = fixture(); + dataset.eval[0].features.insert("world_id".into(), 3); + let report = validate_generated_task(&program, &dataset, &policy); + assert!(report.violations.iter().any(|violation| { + violation.kind == ValidityViolationKind::ForbiddenFeatureLeak + })); + } + + #[test] + fn class_count_contract_catches_missing_or_duplicated_examples() { + let (program, mut dataset, policy) = fixture(); + dataset.eval.pop(); + let report = validate_generated_task(&program, &dataset, &policy); + assert!(report.violations.iter().any(|violation| { + violation.kind == ValidityViolationKind::DeclaredClassCountMismatch + })); + } +} From e6dc71bc4e3bfdc36a35e8be1de28cd2fba2f33a Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:15:41 +0200 Subject: [PATCH 2/8] feat(research): expose benchmark validity infrastructure --- crates/domains/symthaea-psych-bench/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index df0a8b5ea..634ef01f5 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -56,6 +56,8 @@ pub mod benchmarks; pub mod experiment; #[path = "experiment/confirmatory.rs"] pub mod experiment_confirmatory; +#[path = "experiment/validity.rs"] +pub mod experiment_validity; pub mod harness; pub mod substrate_transfer; pub mod wm; From 223ca0c08d20cfff0d23c57a8bceb81f2b484f14 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:17:24 +0200 Subject: [PATCH 3/8] docs(research): preregister 002A3 benchmark validity contract --- .../SYM_ARCH_002A3_BENCHMARK_VALIDITY_V1.md | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 docs/research/SYM_ARCH_002A3_BENCHMARK_VALIDITY_V1.md diff --git a/docs/research/SYM_ARCH_002A3_BENCHMARK_VALIDITY_V1.md b/docs/research/SYM_ARCH_002A3_BENCHMARK_VALIDITY_V1.md new file mode 100644 index 000000000..19b7468e9 --- /dev/null +++ b/docs/research/SYM_ARCH_002A3_BENCHMARK_VALIDITY_V1.md @@ -0,0 +1,149 @@ +# SYM-ARCH-002A3 Benchmark Validity and Mutation Testing v1 + +**Status:** benchmark-integrity infrastructure +**Scientific claim status:** none +**Parent plan:** issue #55 +**Stacked dependency:** PR #57 (`research/sym-arch-002a-core-v1`) + +## Purpose + +SYM-ARCH-002A3 makes benchmark validity a prerequisite for architecture evidence. + +The intended ordering is: + +1. define a typed `TaskProgram`; +2. execute its symbolic oracle over generated examples; +3. validate split identity, labels, support contracts, and leakage policy; +4. deliberately corrupt the benchmark in known ways; +5. require those corruptions to be detected; +6. only then permit architecture scores to enter a ClaimLedger. + +A model result from a benchmark that fails this layer should be treated as **benchmark invalid / inconclusive**, not as evidence that the model succeeded or failed. + +## Executable v1 oracle + +A3 provides an executable interpreter for the v1 `RuleExpr` language: + +- `Eq` / `Ne` compare named integer-valued features; +- `Parity` evaluates a named integer feature modulo a declared modulus; +- `Not`, `And`, `Or`, and `Xor` compose rules recursively. + +Missing required features fail closed. + +`symbolic_oracle_digest` domain-separates and hashes the exact serialized `RuleExpr`. A valid A3 task requires `TaskProgram.oracle_digest` to equal this executable-oracle digest, binding the stated oracle identity to the rule actually used for label checking. + +This v1 contract intentionally supports only `RuleExpr`-grounded symbolic oracles. Richer external simulators/causal worlds should receive their own typed oracle contract rather than bypassing this check. + +## Generated dataset identity + +`GeneratedTaskDataset` binds: + +- the exact `TaskProgram` digest; +- a training example set; +- an evaluation example set. + +Each `ExampleRecord` has: + +- a non-empty example id; +- a deterministic integer feature map (`BTreeMap`); +- explicit support tags; +- the expected oracle label. + +Dataset hashing is set-like for example order: examples are canonicalized by id and support tags are sorted before hashing. Reordering a static train/evaluation set therefore does not manufacture a new dataset identity. + +A separate **feature-only** digest deliberately ignores id, label, and support annotation so a train/eval duplicate cannot evade leakage detection by receiving a new id or label. + +## Validity policy + +`BenchmarkValidityPolicy::task_free_strict()` is the conservative default for latent-context/task-free tests. It rejects features named like explicit task/boundary leakage, including: + +- `__task_id` / `task_id`; +- `__world_id` / `world_id`; +- `boundary_marker`; +- `time_to_switch`. + +The policy also defaults to: + +- feature-disjoint train/evaluation splits; +- declared support tags only; +- both classes represented in both splits. + +Explicit-context experiments may construct a narrower forbidden-key policy, but doing so is an explicit benchmark-design decision rather than silent leakage. + +## Fail-closed checks + +`validate_generated_task` reports independent violation categories rather than one scalar validity score. v1 checks: + +- TaskProgram structural validity; +- dataset ↔ TaskProgram digest binding; +- executable oracle ↔ oracle digest binding; +- non-empty splits; +- example structural validity; +- globally unique example ids; +- feature-identical train/eval leakage; +- forbidden task/world/boundary features; +- split support tags against TaskProgram declarations; +- expected labels against the symbolic oracle; +- observed positive/negative counts against TaskProgram declarations; +- class degeneracy within either split when required by policy. + +Any violation makes the benchmark `invalid`. + +## Mutation testing the validator + +A3 includes deliberate benchmark corruptions: + +1. **flip first evaluation label** — oracle/label integrity failure; +2. **leak first training feature assignment into evaluation** — split leakage; +3. **corrupt program digest** — provenance failure; +4. **inject forbidden task id** — hidden task-identity leakage; +5. **inject undeclared evaluation support** — split/support contract failure. + +`mutation_detection_suite` first requires the unmodified benchmark to validate, then applies every standard corruption independently and requires each mutated benchmark to become invalid. + +This tests the scientific instrument itself rather than assuming its failure paths work. + +## What v1 does not certify + +Passing A3 v1 does **not** prove that a benchmark is free of every shortcut. + +In particular, v1 does not yet automatically detect: + +- a non-reserved feature that happens to encode the label perfectly; +- a marginal-factor shortcut that solves a supposedly relational task; +- nearest-neighbor/template shortcuts; +- distributional artifacts that identify the split; +- semantic equivalence between syntactically different generated programs; +- temporal leakage embedded in otherwise legitimate timestamps; +- task identity hidden through a learned/encoded representation rather than an explicit feature key. + +Those require the planned **construct-validity control ladder**: chance/majority, marginal predictors, nearest-neighbor/lookup, shuffled-relation negative controls, symbolic positive control, and eventually semantic/program-equivalence checks. + +A3 v1 therefore supports the wording: + +> structural/oracle benchmark integrity passed + +not: + +> all possible shortcut learning has been ruled out. + +## Acceptance gate + +The exact stacked PR head should pass: + +1. rustfmt for A3 paths; +2. focused `experiment_validity` tests; +3. `cargo check -p symthaea-psych-bench --lib`; +4. nested boolean-oracle execution test; +5. executable-oracle digest binding test; +6. deterministic order-insensitive dataset identity test; +7. feature-only train/eval leakage test; +8. forbidden task-identity leakage test; +9. declared class-count test; +10. standard mutation suite with every corruption detected. + +## Merge boundary + +This PR is independent of A2 statistics and is stacked directly on #57. It may be reviewed in parallel with #58. + +After #57 merges, retarget/rebase A3 to `main` without changing the validity contract. No architecture result is required for merge because this is scientific instrumentation, not evidence of an architecture advantage. From 99ae17e06c8643c2428e74edf93e859cb9b9bd25 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:17:36 +0200 Subject: [PATCH 4/8] ci(research): gate 002A3 benchmark validity infrastructure --- .github/workflows/sym-arch-002a3-validity.yml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/sym-arch-002a3-validity.yml diff --git a/.github/workflows/sym-arch-002a3-validity.yml b/.github/workflows/sym-arch-002a3-validity.yml new file mode 100644 index 000000000..3840805a2 --- /dev/null +++ b/.github/workflows/sym-arch-002a3-validity.yml @@ -0,0 +1,63 @@ +name: SYM-ARCH-002A3 Benchmark Validity + +on: + pull_request: + branches: + - main + - research/sym-arch-002a-core-v1 + paths: + - '.github/workflows/sym-arch-002a3-validity.yml' + - 'crates/domains/symthaea-psych-bench/src/experiment/validity.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/mod.rs' + - 'crates/domains/symthaea-psych-bench/src/lib.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' + - 'docs/research/SYM_ARCH_002A3_BENCHMARK_VALIDITY_V1.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + benchmark-validity: + name: Validate benchmark oracle and mutation checks + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + with: + components: rustfmt + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-002a3-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-002a3- + + - name: Check A3 Rust formatting + run: | + rustfmt --edition 2024 --check \ + crates/domains/symthaea-psych-bench/src/experiment/validity.rs \ + crates/domains/symthaea-psych-bench/src/lib.rs + + - name: Run benchmark validity and mutation tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_validity:: -- --nocapture + + - name: Check psych-bench library + run: | + cargo check -p symthaea-psych-bench --lib From 79b8d773a1f530489c39e946dcb245c275e3a804 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:55:28 +0200 Subject: [PATCH 5/8] fix(test): remove stale loop trial fixture field --- .../symthaea-psych-bench/src/harness/neuromod_correlation.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs index b4c849fdc..c6056aad3 100644 --- a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs +++ b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs @@ -241,11 +241,6 @@ mod tests { cycle_time_us: 1000 + (i as u64 * 50), learning_occurred: i % 5 == 0, reward: 0.0, - // Added 2026-07-31 to unblock the crate: another session's - // LoopTrialResult gained `cycle_reward` and this test fixture was - // collateral damage. Synthetic data, so 0.0 matches the - // neighbouring `reward`/`moral_score` fixtures. - cycle_reward: 0.0, oxytocin: 0.3, moral_score: 0.0, bath_entropy: 0.5, From 8df8a382fdfe9811437557de02832eae9a05b9c0 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 13:05:49 +0200 Subject: [PATCH 6/8] fix(research): fail closed on missing support provenance --- .../src/experiment/validity.rs | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/experiment/validity.rs b/crates/domains/symthaea-psych-bench/src/experiment/validity.rs index 12f7428b6..a5ee2d0ce 100644 --- a/crates/domains/symthaea-psych-bench/src/experiment/validity.rs +++ b/crates/domains/symthaea-psych-bench/src/experiment/validity.rs @@ -157,7 +157,8 @@ pub struct BenchmarkValidityPolicy { pub forbidden_feature_keys: BTreeSet, /// Reject feature-identical examples across train and evaluation splits. pub require_feature_disjoint_splits: bool, - /// Require every support tag to be declared by the split's TaskProgram support. + /// Require each example to carry at least one support tag and require every + /// support tag to be declared by the split's TaskProgram support. pub require_declared_support_tags: bool, /// Require positive and negative examples in both train and evaluation splits. pub require_both_classes_per_split: bool, @@ -354,12 +355,22 @@ pub fn validate_generated_task( push_violation( &mut violations, ValidityViolationKind::ForbiddenFeatureLeak, - format!("{}:{} contains forbidden feature {key}", split_name, example.example_id), + format!( + "{}:{} contains forbidden feature {key}", + split_name, example.example_id + ), ); } } if policy.require_declared_support_tags { + if example.support_tags.is_empty() { + push_violation( + &mut violations, + ValidityViolationKind::UndeclaredSupport, + format!("{}:{} has no support provenance tag", split_name, example.example_id), + ); + } for tag in &example.support_tags { let normalized = tag.trim().to_string(); if !declared_support.contains(&normalized) { @@ -379,7 +390,10 @@ pub fn validate_generated_task( Ok(label) if label != example.expected_label => push_violation( &mut violations, ValidityViolationKind::OracleMismatch, - format!("{}:{} label disagrees with symbolic oracle", split_name, example.example_id), + format!( + "{}:{} label disagrees with symbolic oracle", + split_name, example.example_id + ), ), Err(error) => push_violation( &mut violations, @@ -420,7 +434,10 @@ pub fn validate_generated_task( push_violation( &mut violations, ValidityViolationKind::TrainEvalFeatureLeak, - format!("evaluation example {} duplicates training features", example.example_id), + format!( + "evaluation example {} duplicates training features", + example.example_id + ), ); } } @@ -472,6 +489,7 @@ pub enum BenchmarkMutation { CorruptProgramDigest, InjectForbiddenTaskId, InjectUndeclaredEvalSupport, + RemoveFirstEvalSupport, } /// Apply an intentionally invalid mutation used to test the validator itself. @@ -517,6 +535,13 @@ pub fn apply_mutation( .ok_or_else(|| "mutation requires a non-empty evaluation split".to_string())?; example.support_tags.push("undeclared-support".into()); } + BenchmarkMutation::RemoveFirstEvalSupport => { + let example = mutated + .eval + .first_mut() + .ok_or_else(|| "mutation requires a non-empty evaluation split".to_string())?; + example.support_tags.clear(); + } } Ok(mutated) } @@ -545,6 +570,7 @@ pub fn mutation_detection_suite( BenchmarkMutation::CorruptProgramDigest, BenchmarkMutation::InjectForbiddenTaskId, BenchmarkMutation::InjectUndeclaredEvalSupport, + BenchmarkMutation::RemoveFirstEvalSupport, ]; let mut detections = Vec::with_capacity(mutations.len()); @@ -676,7 +702,7 @@ mod tests { fn mutation_suite_detects_every_standard_benchmark_corruption() { let (program, dataset, policy) = fixture(); let detections = mutation_detection_suite(&program, &dataset, &policy).unwrap(); - assert_eq!(detections.len(), 5); + assert_eq!(detections.len(), 6); assert!(detections.iter().all(|detection| detection.detected)); } @@ -690,6 +716,16 @@ mod tests { })); } + #[test] + fn missing_support_provenance_fails_closed() { + let (program, mut dataset, policy) = fixture(); + dataset.eval[0].support_tags.clear(); + let report = validate_generated_task(&program, &dataset, &policy); + assert!(report.violations.iter().any(|violation| { + violation.kind == ValidityViolationKind::UndeclaredSupport + })); + } + #[test] fn class_count_contract_catches_missing_or_duplicated_examples() { let (program, mut dataset, policy) = fixture(); From 60607533fa38104f8d1389426a6e8eae3c27acca Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 15:39:35 +0200 Subject: [PATCH 7/8] ci(research): supersede stale A3 validation runs --- .github/workflows/sym-arch-002a3-validity.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/sym-arch-002a3-validity.yml b/.github/workflows/sym-arch-002a3-validity.yml index 3840805a2..46e40e817 100644 --- a/.github/workflows/sym-arch-002a3-validity.yml +++ b/.github/workflows/sym-arch-002a3-validity.yml @@ -16,6 +16,12 @@ on: - 'Cargo.lock' workflow_dispatch: +concurrency: + # Automatic branch runs supersede stale checks. Deliberate manual validation + # runs are unique and therefore never cancel one another. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + permissions: contents: read From a942eda65c5ef9ca109909d61505a468e8c1c366 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Wed, 26 Aug 2026 09:28:51 +0200 Subject: [PATCH 8/8] ci(research): defer A3 dedicated gate while draft --- .github/workflows/sym-arch-002a3-validity.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sym-arch-002a3-validity.yml b/.github/workflows/sym-arch-002a3-validity.yml index 46e40e817..cdab2ffa5 100644 --- a/.github/workflows/sym-arch-002a3-validity.yml +++ b/.github/workflows/sym-arch-002a3-validity.yml @@ -2,6 +2,7 @@ name: SYM-ARCH-002A3 Benchmark Validity on: pull_request: + types: [opened, synchronize, reopened, ready_for_review] branches: - main - research/sym-arch-002a-core-v1 @@ -32,6 +33,9 @@ env: jobs: benchmark-validity: name: Validate benchmark oracle and mutation checks + # Draft stacked PRs cannot merge and should not consume scarce runner capacity. + # Promoting to ready-for-review explicitly triggers this exact gate. + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest timeout-minutes: 20