From ee5752f619f98b69c31b4cbf1eb0a911a1ae52a2 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Mon, 24 Aug 2026 23:06:24 +0200 Subject: [PATCH 01/12] feat(psych-bench): add SYM-ARCH-001 discrimination campaign --- .../src/benchmarks/architecture.rs | 803 ++++++++++++++++++ 1 file changed, 803 insertions(+) create mode 100644 crates/domains/symthaea-psych-bench/src/benchmarks/architecture.rs diff --git a/crates/domains/symthaea-psych-bench/src/benchmarks/architecture.rs b/crates/domains/symthaea-psych-bench/src/benchmarks/architecture.rs new file mode 100644 index 0000000000..067420f449 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/benchmarks/architecture.rs @@ -0,0 +1,803 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! SYM-ARCH-001: continual compositional adaptation architecture discrimination. +//! +//! This benchmark is deliberately narrower than a "general intelligence" claim. +//! It asks whether mechanisms already present in Symthaea buy measurable value on +//! three pre-registered phenomena under one deterministic task family: +//! +//! 1. retention across sequentially learned relational worlds, +//! 2. generalization to held-out factor combinations, and +//! 3. adaptation after a contingency reversal. +//! +//! The candidate is an HDC-LTC representation with Hebbian plasticity and a +//! common associative prototype readout. Ablations/controls are: +//! +//! - online linear SGD (simple conventional control), +//! - vanilla HDC + the same prototype readout, +//! - a fixed diagonal SSM + the same prototype readout, and +//! - HDC-LTC with liquid dynamics but frozen HDC-LTC weights. +//! +//! This is a *mechanism-level* benchmark. It does NOT exercise the full live +//! `CognitiveLoopService`, and it must not be reported as a frontier-model or +//! full-Symthaea comparison. In particular, it intentionally bypasses the old +//! reward-driven `cycle_with_hv()` live harness path whose reward-consumption gap +//! was frozen in `SYMTHAEA_UAL_LIVE_DIAGNOSTIC_P1_COLLAPSE_TRACE_2026-07-30.md`. + +use serde::{Deserialize, Serialize}; +use std::time::Instant; +use symthaea_core::hdc::ContinuousHV; +use symthaea_core::hdc::hdc_ltc_unified::{HdcLtcUnifiedNeuron, UnifiedConfig}; +use symthaea_ssm::{SelectiveParams, SsmState}; + +const WORLDS: usize = 4; +const VALUES: usize = 4; +const HELD_OUT_PER_WORLD: usize = 4; +const REVERSAL_WINDOW: usize = 32; + +/// Pre-registered experiment configuration. Defaults are the PR campaign. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SymArch001Config { + pub dimension: usize, + pub seeds: usize, + pub train_epochs_per_world: usize, + pub reversal_epochs: usize, + pub prototype_alpha: f32, + pub hebbian_learning_rate: f32, + pub win_margin: f64, + pub regression_tolerance: f64, +} + +impl Default for SymArch001Config { + fn default() -> Self { + Self { + dimension: 512, + seeds: 16, + train_epochs_per_world: 16, + reversal_epochs: 12, + prototype_alpha: 0.15, + hebbian_learning_rate: 0.002, + win_margin: 0.05, + regression_tolerance: 0.05, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SeedMetrics { + pub seed: u64, + /// Mean accuracy over each world's *trained* combinations after all worlds. + pub final_retention_accuracy: f64, + /// Mean accuracy over balanced held-out combinations after all worlds. + pub heldout_compositional_accuracy: f64, + /// Mean peak-to-final drop on the trained combinations. Lower is better. + pub mean_forgetting: f64, + /// Accuracy on the final 32 pre-update predictions after the rule is inverted. + pub reversal_final_accuracy: f64, + /// First trial index whose trailing 32-trial window reaches >= 75%. + /// `reversal_trials + 1` means the criterion was never reached. + pub reversal_adaptation_latency: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MeanCi95 { + pub mean: f64, + pub ci95_low: f64, + pub ci95_high: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AgentSummary { + pub agent: String, + pub retention: MeanCi95, + pub compositional: MeanCi95, + pub forgetting: MeanCi95, + pub reversal_final: MeanCi95, + pub reversal_latency: MeanCi95, + pub per_seed: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DecisionEvidence { + pub verdict: String, + pub candidate: String, + pub retention_delta_vs_best_control: f64, + pub compositional_delta_vs_best_control: f64, + pub reversal_delta_vs_best_control: f64, + pub forgetting_delta_vs_best_control: f64, + pub target_wins_at_margin: usize, + pub target_regressions_beyond_tolerance: usize, + pub preregistered_rule: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResourceEvidence { + pub wall_time_ms: u128, + pub observations_per_agent: usize, + pub representation_dimension: usize, + pub ssm_state_per_dimension: usize, + pub note: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SymArch001Report { + pub schema: String, + pub source_revision: Option, + pub scope: String, + pub config: SymArch001Config, + pub agents: Vec, + pub decision: DecisionEvidence, + pub resources: ResourceEvidence, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Pair { + i: usize, + j: usize, +} + +fn rule(world: usize, p: Pair) -> bool { + match world { + // Balanced relational rules: 8 positive / 8 negative each. + 0 => (p.i & 1) == (p.j & 1), + 1 => ((p.i + p.j) & 3) < 2, + 2 => ((p.i + VALUES - p.j) & 3) < 2, + 3 => ((p.i ^ p.j) & 2) == 0, + _ => unreachable!("world index is bounded by WORLDS"), + } +} + +fn held_out(world: usize) -> Vec { + // Deterministically take the first two positives and first two negatives. + // This makes both the training set (6/6) and held-out set (2/2) balanced. + let mut positives = Vec::with_capacity(2); + let mut negatives = Vec::with_capacity(2); + for i in 0..VALUES { + for j in 0..VALUES { + let p = Pair { i, j }; + if rule(world, p) { + if positives.len() < 2 { + positives.push(p); + } + } else if negatives.len() < 2 { + negatives.push(p); + } + } + } + positives.extend(negatives); + debug_assert_eq!(positives.len(), HELD_OUT_PER_WORLD); + positives +} + +fn training_pairs(world: usize) -> Vec { + let held = held_out(world); + let mut out = Vec::with_capacity(VALUES * VALUES - HELD_OUT_PER_WORLD); + for i in 0..VALUES { + for j in 0..VALUES { + let p = Pair { i, j }; + if !held.contains(&p) { + out.push(p); + } + } + } + out +} + +#[derive(Clone)] +struct TaskSpace { + values: [ContinuousHV; VALUES], + worlds: [ContinuousHV; WORLDS], + shape_role: ContinuousHV, + texture_role: ContinuousHV, +} + +impl TaskSpace { + fn new(dim: usize, seed: u64) -> Self { + let values = std::array::from_fn(|i| { + ContinuousHV::random(dim, mix_seed(seed, 0x1000 + i as u64)) + }); + let worlds = std::array::from_fn(|i| { + ContinuousHV::random(dim, mix_seed(seed, 0x2000 + i as u64)) + }); + Self { + values, + worlds, + shape_role: ContinuousHV::random(dim, mix_seed(seed, 0x3001)), + texture_role: ContinuousHV::random(dim, mix_seed(seed, 0x3002)), + } + } + + fn encode(&self, world: usize, p: Pair) -> ContinuousHV { + let shape = self.values[p.i].bind(&self.shape_role); + let texture = self.values[p.j].bind(&self.texture_role); + shape + .bind(&texture) + .bind(&self.worlds[world]) + .normalize() + } +} + +#[derive(Debug, Clone)] +struct PrototypeMemory { + negative: ContinuousHV, + positive: ContinuousHV, + negative_seen: usize, + positive_seen: usize, + alpha: f32, +} + +impl PrototypeMemory { + fn new(dim: usize, alpha: f32) -> Self { + Self { + negative: ContinuousHV::zero(dim), + positive: ContinuousHV::zero(dim), + negative_seen: 0, + positive_seen: 0, + alpha, + } + } + + fn update(&mut self, representation: &ContinuousHV, label: bool) { + let (slot, seen) = if label { + (&mut self.positive, &mut self.positive_seen) + } else { + (&mut self.negative, &mut self.negative_seen) + }; + if *seen == 0 { + *slot = representation.normalize(); + } else { + let old = slot.clone(); + *slot = ContinuousHV::weighted_bundle( + &[&old, representation], + &[1.0 - self.alpha, self.alpha], + ) + .normalize(); + } + *seen += 1; + } + + fn predict(&self, representation: &ContinuousHV) -> bool { + match (self.negative_seen, self.positive_seen) { + (0, 0) => false, + (0, _) => true, + (_, 0) => false, + _ => { + representation.similarity(&self.positive) + > representation.similarity(&self.negative) + } + } + } +} + +#[derive(Debug, Clone)] +struct LinearSgd { + weights: Vec, + bias: f32, + learning_rate: f32, +} + +impl LinearSgd { + fn new(dim: usize) -> Self { + Self { + weights: vec![0.0; dim], + bias: 0.0, + learning_rate: 0.08, + } + } + + fn logit(&self, x: &ContinuousHV) -> f32 { + self.bias + + self + .weights + .iter() + .zip(x.values.iter()) + .map(|(w, v)| w * v) + .sum::() + } + + fn predict(&self, x: &ContinuousHV) -> bool { + self.logit(x) >= 0.0 + } + + fn observe(&mut self, x: &ContinuousHV, label: bool) { + let z = self.logit(x).clamp(-20.0, 20.0); + let p = 1.0 / (1.0 + (-z).exp()); + let target = if label { 1.0 } else { 0.0 }; + let error = target - p; + for (w, v) in self.weights.iter_mut().zip(x.values.iter()) { + *w += self.learning_rate * error * *v; + } + self.bias += self.learning_rate * error; + } +} + +#[derive(Clone)] +enum Agent { + Linear(LinearSgd), + Vanilla { + memory: PrototypeMemory, + }, + Ssm { + params: SelectiveParams, + state: SsmState, + memory: PrototypeMemory, + dim: usize, + }, + LiquidFrozen { + liquid: HdcLtcUnifiedNeuron, + memory: PrototypeMemory, + }, + LiquidHebbian { + liquid: HdcLtcUnifiedNeuron, + memory: PrototypeMemory, + hebbian_lr: f32, + }, +} + +impl Agent { + fn all(config: &SymArch001Config, seed: u64) -> Vec { + let dim = config.dimension; + let liquid_config = UnifiedConfig { + dimension: dim, + learning_rate: config.hebbian_learning_rate, + ..Default::default() + }; + vec![ + Self::Linear(LinearSgd::new(dim)), + Self::Vanilla { + memory: PrototypeMemory::new(dim, config.prototype_alpha), + }, + Self::Ssm { + params: SelectiveParams::new(dim, 2), + state: SsmState::new(dim, 2), + memory: PrototypeMemory::new(dim, config.prototype_alpha), + dim, + }, + Self::LiquidFrozen { + liquid: HdcLtcUnifiedNeuron::new(liquid_config.clone(), mix_seed(seed, 0xA001)), + memory: PrototypeMemory::new(dim, config.prototype_alpha), + }, + Self::LiquidHebbian { + liquid: HdcLtcUnifiedNeuron::new(liquid_config, mix_seed(seed, 0xA002)), + memory: PrototypeMemory::new(dim, config.prototype_alpha), + hebbian_lr: config.hebbian_learning_rate, + }, + ] + } + + fn name(&self) -> &'static str { + match self { + Self::Linear(_) => "linear_sgd", + Self::Vanilla { .. } => "vanilla_hdc", + Self::Ssm { .. } => "fixed_diagonal_ssm", + Self::LiquidFrozen { .. } => "hdc_ltc_frozen", + Self::LiquidHebbian { .. } => "hdc_ltc_hebbian", + } + } + + fn predict(&self, input: &ContinuousHV) -> bool { + match self { + Self::Linear(linear) => linear.predict(input), + Self::Vanilla { memory } => memory.predict(input), + Self::Ssm { + params, + state, + memory, + dim, + } => { + let mut shadow = state.clone(); + let mut output = vec![0.0_f32; *dim]; + shadow.step(&input.values, params, &mut output); + let representation = ContinuousHV::from_values(output).normalize(); + memory.predict(&representation) + } + Self::LiquidFrozen { liquid, memory } + | Self::LiquidHebbian { liquid, memory, .. } => { + let mut shadow = liquid.clone(); + shadow.evolve_closed_form(0.05, input); + memory.predict(shadow.state()) + } + } + } + + fn observe(&mut self, input: &ContinuousHV, label: bool) { + match self { + Self::Linear(linear) => linear.observe(input, label), + Self::Vanilla { memory } => memory.update(input, label), + Self::Ssm { + params, + state, + memory, + dim, + } => { + let mut output = vec![0.0_f32; *dim]; + state.step(&input.values, params, &mut output); + let representation = ContinuousHV::from_values(output).normalize(); + memory.update(&representation, label); + } + Self::LiquidFrozen { liquid, memory } => { + liquid.evolve_closed_form(0.05, input); + let representation = liquid.state().clone(); + memory.update(&representation, label); + } + Self::LiquidHebbian { + liquid, + memory, + hebbian_lr, + } => { + liquid.evolve_closed_form(0.05, input); + let representation = liquid.state().clone(); + memory.update(&representation, label); + liquid.hebbian_update(input, Some(*hebbian_lr)); + } + } + } +} + +fn accuracy(agent: &Agent, task: &TaskSpace, world: usize, pairs: &[Pair], invert: bool) -> f64 { + let correct = pairs + .iter() + .filter(|&&p| { + let target = rule(world, p) ^ invert; + agent.predict(&task.encode(world, p)) == target + }) + .count(); + correct as f64 / pairs.len() as f64 +} + +fn shuffled_pairs(world: usize, seed: u64, epoch: usize) -> Vec { + let mut pairs = training_pairs(world); + let mut rng = mix_seed(seed, 0x5000 + world as u64 * 257 + epoch as u64); + for i in (1..pairs.len()).rev() { + let j = (next_u64(&mut rng) as usize) % (i + 1); + pairs.swap(i, j); + } + pairs +} + +fn run_seed(config: &SymArch001Config, seed: u64) -> Vec<(String, SeedMetrics)> { + let task = TaskSpace::new(config.dimension, seed); + let mut agents = Agent::all(config, seed); + let mut peaks = vec![[0.0_f64; WORLDS]; agents.len()]; + + // Sequential worlds A -> B -> C -> D, with no replay of old worlds. + for world in 0..WORLDS { + for epoch in 0..config.train_epochs_per_world { + for p in shuffled_pairs(world, seed, epoch) { + let x = task.encode(world, p); + let y = rule(world, p); + for agent in &mut agents { + agent.observe(&x, y); + } + } + } + + // Evaluate every world seen so far, but do not mutate agent state. + for (agent_idx, agent) in agents.iter().enumerate() { + for seen_world in 0..=world { + let score = accuracy(agent, &task, seen_world, &training_pairs(seen_world), false); + peaks[agent_idx][seen_world] = peaks[agent_idx][seen_world].max(score); + } + } + } + + let all_pairs: Vec = (0..VALUES) + .flat_map(|i| (0..VALUES).map(move |j| Pair { i, j })) + .collect(); + let reversal_trials = config.reversal_epochs * all_pairs.len(); + + agents + .iter() + .enumerate() + .map(|(agent_idx, agent)| { + let final_by_world: Vec = (0..WORLDS) + .map(|world| accuracy(agent, &task, world, &training_pairs(world), false)) + .collect(); + let heldout_by_world: Vec = (0..WORLDS) + .map(|world| accuracy(agent, &task, world, &held_out(world), false)) + .collect(); + let forgetting: Vec = (0..WORLDS) + .map(|world| (peaks[agent_idx][world] - final_by_world[world]).max(0.0)) + .collect(); + + // Isolate the reversal from the retention/composition measurement. + // Prediction is scored *before* each supervised update. + let mut reversed = agent.clone(); + let mut outcomes = Vec::with_capacity(reversal_trials); + let mut adaptation_latency = reversal_trials + 1; + for epoch in 0..config.reversal_epochs { + let mut order = all_pairs.clone(); + let mut rng = mix_seed(seed, 0x9000 + epoch as u64); + for i in (1..order.len()).rev() { + let j = (next_u64(&mut rng) as usize) % (i + 1); + order.swap(i, j); + } + for p in order { + let x = task.encode(WORLDS - 1, p); + let target = !rule(WORLDS - 1, p); + let correct = reversed.predict(&x) == target; + outcomes.push(correct); + reversed.observe(&x, target); + if outcomes.len() >= REVERSAL_WINDOW && adaptation_latency > reversal_trials { + let tail = &outcomes[outcomes.len() - REVERSAL_WINDOW..]; + let rolling = tail.iter().filter(|&&v| v).count() as f64 + / REVERSAL_WINDOW as f64; + if rolling >= 0.75 { + adaptation_latency = outcomes.len(); + } + } + } + } + let tail_len = REVERSAL_WINDOW.min(outcomes.len()); + let reversal_final = outcomes[outcomes.len() - tail_len..] + .iter() + .filter(|&&v| v) + .count() as f64 + / tail_len as f64; + + ( + agent.name().to_string(), + SeedMetrics { + seed, + final_retention_accuracy: mean(&final_by_world), + heldout_compositional_accuracy: mean(&heldout_by_world), + mean_forgetting: mean(&forgetting), + reversal_final_accuracy: reversal_final, + reversal_adaptation_latency: adaptation_latency, + }, + ) + }) + .collect() +} + +/// Run the pre-registered SYM-ARCH-001 campaign. +pub fn run_sym_arch_001( + config: SymArch001Config, + source_revision: Option, +) -> SymArch001Report { + assert!(config.dimension >= 32, "dimension must be >= 32"); + assert!(config.seeds > 0, "at least one seed is required"); + assert!(config.train_epochs_per_world > 0); + assert!(config.reversal_epochs > 0); + + let started = Instant::now(); + let mut by_agent: Vec<(String, Vec)> = Vec::new(); + for seed_idx in 0..config.seeds { + let seed = mix_seed(0x5A17_2026_0000_0001, seed_idx as u64); + for (name, metrics) in run_seed(&config, seed) { + if let Some((_, rows)) = by_agent.iter_mut().find(|(n, _)| n == &name) { + rows.push(metrics); + } else { + by_agent.push((name, vec![metrics])); + } + } + } + + let agents: Vec = by_agent + .into_iter() + .map(|(agent, rows)| AgentSummary { + retention: mean_ci95(rows.iter().map(|r| r.final_retention_accuracy)), + compositional: mean_ci95(rows.iter().map(|r| r.heldout_compositional_accuracy)), + forgetting: mean_ci95(rows.iter().map(|r| r.mean_forgetting)), + reversal_final: mean_ci95(rows.iter().map(|r| r.reversal_final_accuracy)), + reversal_latency: mean_ci95( + rows.iter() + .map(|r| r.reversal_adaptation_latency as f64), + ), + agent, + per_seed: rows, + }) + .collect(); + + let decision = classify(&agents, &config); + let train_pairs_per_world = VALUES * VALUES - HELD_OUT_PER_WORLD; + let observations_per_agent = WORLDS * config.train_epochs_per_world * train_pairs_per_world + + config.reversal_epochs * VALUES * VALUES; + + SymArch001Report { + schema: "symthaea.sym-arch-001.v1".to_string(), + source_revision, + scope: "mechanism-level HDC/HDC-LTC/SSM/linear discrimination; not full live Symthaea" + .to_string(), + config: config.clone(), + agents, + decision, + resources: ResourceEvidence { + wall_time_ms: started.elapsed().as_millis(), + observations_per_agent, + representation_dimension: config.dimension, + ssm_state_per_dimension: 2, + note: "Wall time is observational and runner-dependent; sample counts and dimensions are the reproducible resource-normalization anchors.".to_string(), + }, + } +} + +fn classify(agents: &[AgentSummary], config: &SymArch001Config) -> DecisionEvidence { + let candidate_name = "hdc_ltc_hebbian"; + let candidate = agents + .iter() + .find(|a| a.agent == candidate_name) + .expect("candidate summary must exist"); + let controls: Vec<&AgentSummary> = agents + .iter() + .filter(|a| a.agent != candidate_name) + .collect(); + + let best_retention = controls + .iter() + .map(|a| a.retention.mean) + .fold(f64::NEG_INFINITY, f64::max); + let best_composition = controls + .iter() + .map(|a| a.compositional.mean) + .fold(f64::NEG_INFINITY, f64::max); + let best_reversal = controls + .iter() + .map(|a| a.reversal_final.mean) + .fold(f64::NEG_INFINITY, f64::max); + let best_forgetting = controls + .iter() + .map(|a| a.forgetting.mean) + .fold(f64::INFINITY, f64::min); + + let retention_delta = candidate.retention.mean - best_retention; + let composition_delta = candidate.compositional.mean - best_composition; + let reversal_delta = candidate.reversal_final.mean - best_reversal; + // Positive is better: control forgetting minus candidate forgetting. + let forgetting_delta = best_forgetting - candidate.forgetting.mean; + + let targets = [retention_delta, composition_delta, reversal_delta]; + let wins = targets + .iter() + .filter(|&&d| d >= config.win_margin) + .count(); + let regressions = targets + .iter() + .filter(|&&d| d < -config.regression_tolerance) + .count(); + let forgetting_regression = forgetting_delta < -config.regression_tolerance; + + let verdict = if wins >= 2 && regressions == 0 && !forgetting_regression { + "PASS" + } else if regressions >= 2 || forgetting_regression { + "NEGATIVE" + } else if wins >= 1 && regressions == 0 { + "MIXED" + } else { + "NULL" + }; + + DecisionEvidence { + verdict: verdict.to_string(), + candidate: candidate_name.to_string(), + retention_delta_vs_best_control: retention_delta, + compositional_delta_vs_best_control: composition_delta, + reversal_delta_vs_best_control: reversal_delta, + forgetting_delta_vs_best_control: forgetting_delta, + target_wins_at_margin: wins, + target_regressions_beyond_tolerance: regressions + usize::from(forgetting_regression), + preregistered_rule: format!( + "PASS iff candidate beats the strongest control by >= {:.2} on at least two of retention/composition/reversal, loses by no more than {:.2} on the other target(s), and forgetting is not worse than the best control by > {:.2}; MIXED requires >=1 target win with no target regressions; NEGATIVE requires >=2 target regressions or a forgetting regression; otherwise NULL.", + config.win_margin, config.regression_tolerance, config.regression_tolerance + ), + } +} + +fn mean(values: &[f64]) -> f64 { + values.iter().sum::() / values.len() as f64 +} + +fn mean_ci95(values: I) -> MeanCi95 +where + I: IntoIterator, +{ + let values: Vec = values.into_iter().collect(); + let m = mean(&values); + if values.len() < 2 { + return MeanCi95 { + mean: m, + ci95_low: m, + ci95_high: m, + }; + } + let variance = values + .iter() + .map(|v| (v - m).powi(2)) + .sum::() + / (values.len() - 1) as f64; + let half = 1.96 * variance.sqrt() / (values.len() as f64).sqrt(); + MeanCi95 { + mean: m, + ci95_low: (m - half).max(0.0), + ci95_high: m + half, + } +} + +fn mix_seed(seed: u64, salt: u64) -> u64 { + let mut x = seed ^ salt.wrapping_mul(0x9E37_79B9_7F4A_7C15); + // SplitMix64 finalizer. + x ^= x >> 30; + x = x.wrapping_mul(0xBF58_476D_1CE4_E5B9); + x ^= x >> 27; + x = x.wrapping_mul(0x94D0_49BB_1331_11EB); + x ^ (x >> 31) +} + +fn next_u64(state: &mut u64) -> u64 { + let mut x = *state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + *state = x; + x +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn task_splits_are_balanced_and_disjoint() { + for world in 0..WORLDS { + let held = held_out(world); + let train = training_pairs(world); + assert_eq!(held.len(), 4); + assert_eq!(train.len(), 12); + assert!(held.iter().all(|p| !train.contains(p))); + assert_eq!(held.iter().filter(|&&p| rule(world, p)).count(), 2); + assert_eq!(train.iter().filter(|&&p| rule(world, p)).count(), 6); + } + } + + #[test] + fn smoke_campaign_produces_finite_metrics() { + let report = run_sym_arch_001( + SymArch001Config { + dimension: 64, + seeds: 1, + train_epochs_per_world: 1, + reversal_epochs: 2, + ..Default::default() + }, + None, + ); + assert_eq!(report.agents.len(), 5); + for agent in &report.agents { + for value in [ + agent.retention.mean, + agent.compositional.mean, + agent.forgetting.mean, + agent.reversal_final.mean, + agent.reversal_latency.mean, + ] { + assert!(value.is_finite(), "{} emitted non-finite metric", agent.agent); + } + } + assert!(matches!( + report.decision.verdict.as_str(), + "PASS" | "MIXED" | "NULL" | "NEGATIVE" + )); + } + + #[test] + fn fixed_seed_is_metric_deterministic() { + let cfg = SymArch001Config { + dimension: 64, + seeds: 1, + train_epochs_per_world: 1, + reversal_epochs: 2, + ..Default::default() + }; + let a = run_sym_arch_001(cfg.clone(), None); + let b = run_sym_arch_001(cfg, None); + for (left, right) in a.agents.iter().zip(b.agents.iter()) { + assert_eq!(left.agent, right.agent); + assert_eq!(left.retention.mean, right.retention.mean); + assert_eq!(left.compositional.mean, right.compositional.mean); + assert_eq!(left.forgetting.mean, right.forgetting.mean); + assert_eq!(left.reversal_final.mean, right.reversal_final.mean); + assert_eq!(left.reversal_latency.mean, right.reversal_latency.mean); + } + } +} From 9e8c96e32be3cebbba77f9df073395b9ed68eeef Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Mon, 24 Aug 2026 23:06:39 +0200 Subject: [PATCH 02/12] feat(psych-bench): expose architecture benchmarks --- crates/domains/symthaea-psych-bench/src/benchmarks/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/domains/symthaea-psych-bench/src/benchmarks/mod.rs b/crates/domains/symthaea-psych-bench/src/benchmarks/mod.rs index 815f3c3840..76a723a728 100644 --- a/crates/domains/symthaea-psych-bench/src/benchmarks/mod.rs +++ b/crates/domains/symthaea-psych-bench/src/benchmarks/mod.rs @@ -4,6 +4,7 @@ //! Psychological benchmark implementations. pub mod affect; +pub mod architecture; pub mod attention; pub mod binding; pub mod butlin; From 1e976c7f1132db828a1aa7c369f283393cdbc42d Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Mon, 24 Aug 2026 23:06:50 +0200 Subject: [PATCH 03/12] feat(psych-bench): add SYM-ARCH-001 evidence runner --- .../src/bin/sym_arch_001.rs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 crates/domains/symthaea-psych-bench/src/bin/sym_arch_001.rs diff --git a/crates/domains/symthaea-psych-bench/src/bin/sym_arch_001.rs b/crates/domains/symthaea-psych-bench/src/bin/sym_arch_001.rs new file mode 100644 index 0000000000..92c97d55b8 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/bin/sym_arch_001.rs @@ -0,0 +1,52 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::env; +use std::fs; +use std::path::PathBuf; +use symthaea_psych_bench::benchmarks::architecture::{SymArch001Config, run_sym_arch_001}; + +fn main() { + let output = parse_output_path(); + if let Some(parent) = output.parent() { + fs::create_dir_all(parent).expect("failed to create SYM-ARCH-001 output directory"); + } + + let source_revision = env::var("GITHUB_SHA") + .ok() + .or_else(|| env::var("SYMTHAEA_SOURCE_REVISION").ok()); + let report = run_sym_arch_001(SymArch001Config::default(), source_revision); + let json = serde_json::to_string_pretty(&report).expect("failed to serialize SYM-ARCH-001 report"); + fs::write(&output, &json).expect("failed to write SYM-ARCH-001 report"); + + println!("SYM-ARCH-001 verdict: {}", report.decision.verdict); + println!("evidence: {}", output.display()); + for agent in &report.agents { + println!( + "{:<20} retention={:.3} composition={:.3} forgetting={:.3} reversal={:.3} latency={:.1}", + agent.agent, + agent.retention.mean, + agent.compositional.mean, + agent.forgetting.mean, + agent.reversal_final.mean, + agent.reversal_latency.mean, + ); + } + println!( + "candidate deltas: retention={:+.3}, composition={:+.3}, reversal={:+.3}, forgetting={:+.3}", + report.decision.retention_delta_vs_best_control, + report.decision.compositional_delta_vs_best_control, + report.decision.reversal_delta_vs_best_control, + report.decision.forgetting_delta_vs_best_control, + ); +} + +fn parse_output_path() -> PathBuf { + let mut args = env::args().skip(1); + while let Some(arg) = args.next() { + if arg == "--output" { + return PathBuf::from(args.next().expect("--output requires a path")); + } + } + PathBuf::from("artifacts/sym-arch-001/report.json") +} From edefc8a90767f64eb34701bfad0a5fde5db25d0b Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Mon, 24 Aug 2026 23:07:19 +0200 Subject: [PATCH 04/12] docs(research): preregister SYM-ARCH-001 --- ...SYM_ARCH_001_PREREGISTRATION_2026-08-24.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 docs/research/SYM_ARCH_001_PREREGISTRATION_2026-08-24.md diff --git a/docs/research/SYM_ARCH_001_PREREGISTRATION_2026-08-24.md b/docs/research/SYM_ARCH_001_PREREGISTRATION_2026-08-24.md new file mode 100644 index 0000000000..205821ca26 --- /dev/null +++ b/docs/research/SYM_ARCH_001_PREREGISTRATION_2026-08-24.md @@ -0,0 +1,143 @@ +# SYM-ARCH-001 — Continual Compositional Adaptation + +**Status:** preregistered experiment design, to be executed by the pull-request workflow before interpreting the result. + +## Research question + +Does the current Symthaea HDC-LTC mechanism provide a practically meaningful advantage over simpler controls on a shared continual-learning task family when data, dimensionality, update opportunities, and evaluation splits are held fixed? + +This experiment is intentionally narrower than any claim about AGI, consciousness, language-model capability, or full live Symthaea. It is a mechanism-level architecture discrimination test. + +## Why this is a new work unit + +The frozen UAL-P4a-v2 result tested a benchmark-local HDC learner and did **not** demonstrate held-out compositional generalization in its qualified blocked arm. That result remains untouched. + +The earlier live UAL-P1 reversal diagnostic also remains untouched. Its follow-up trace established that the `CognitiveLoopBenchmarkRunner` path based on `cycle_with_hv()` does not consume `provide_reward()` and therefore cannot support a confirmatory reward-driven reversal claim without separate production-harness work. + +SYM-ARCH-001 therefore bypasses that invalid reward path and uses direct supervised online updates for every compared mechanism. + +## Systems under test + +All systems see the exact same deterministic encoded inputs in the exact same order for each seed. + +1. `linear_sgd` — online linear logistic/SGD control. +2. `vanilla_hdc` — raw compositional HDC representation with an associative prototype readout. +3. `fixed_diagonal_ssm` — the existing Symthaea diagonal SSM state transform with the same associative prototype readout. +4. `hdc_ltc_frozen` — HDC-LTC temporal dynamics with frozen HDC-LTC weights and the same prototype readout. +5. `hdc_ltc_hebbian` — **candidate**: HDC-LTC temporal dynamics plus existing Hebbian plasticity and the same prototype readout. + +The experiment does **not** claim that `linear_sgd` or `fixed_diagonal_ssm` is a frontier transformer/GRU/SSM baseline. A later external-baseline tranche is required before making industry-level efficiency claims. + +## Task family + +There are four sequential relational worlds, each operating on two four-valued factors. Inputs bind: + +- a shared value hypervector, +- a shape role, +- a texture role, and +- a world context. + +Each world defines a deterministic balanced binary relation over the 16 possible factor pairs. For every world: + +- 12 combinations are training items: 6 positive, 6 negative; +- 4 combinations are held out: 2 positive, 2 negative; +- held-out items never appear during that world's training. + +Worlds are learned strictly A → B → C → D. There is **no replay** of prior worlds while a later world is trained. + +## Primary phenomena + +### 1. Continual retention + +After all four worlds are learned, compute mean accuracy on each world's 12 trained combinations. + +Primary metric: `final_retention_accuracy`. + +### 2. Compositional transfer + +After all four worlds are learned, score the four balanced held-out combinations from every world. + +Primary metric: `heldout_compositional_accuracy`. + +### 3. Forgetting + +After each world phase, evaluate every world seen so far without mutating agent state. For each world record its best observed trained-set accuracy, then subtract its final trained-set accuracy. + +Primary metric: `mean_forgetting` (lower is better). + +### 4. Contingency reversal + +Clone the post-training state so reversal cannot contaminate the retention/composition measurements. In world D, invert the relation labels and present all 16 combinations repeatedly in deterministic shuffled order. + +Predictions are scored **before** each supervised update. + +Metrics: + +- final accuracy over the trailing 32 predictions; +- first trial whose trailing 32-trial window reaches at least 75% accuracy; if never reached, report `reversal_trials + 1`. + +This is a contingency-reversal test, not yet a full causal-intervention/do-calculus test. + +## Fixed campaign configuration + +- representation dimension: **512**; +- seeds: **16**; +- training epochs per world: **16**; +- reversal epochs: **12**; +- prototype update alpha: **0.15**; +- candidate Hebbian learning rate: **0.002**; +- practical win margin: **0.05** absolute accuracy; +- tolerated regression margin: **0.05** absolute accuracy. + +Changing any of these after seeing the PR result creates a new experiment version rather than modifying the interpretation of SYM-ARCH-001 v1. + +## Decision rule + +Candidate: `hdc_ltc_hebbian`. + +For retention, composition, and reversal-final accuracy, compare the candidate mean against the **strongest control mean for that metric**. + +- **PASS:** candidate wins by at least 0.05 on at least two of the three target phenomena, has no target regression worse than 0.05, and mean forgetting is not worse than the best control by more than 0.05. +- **MIXED:** at least one target win of 0.05 or more, no target regression worse than 0.05, and no forgetting regression beyond tolerance. +- **NEGATIVE:** at least two target regressions worse than 0.05, or forgetting is worse than the best control by more than 0.05. +- **NULL:** all other outcomes. + +The workflow reports this verdict mechanically. The threshold is not changed after results are observed. + +## Statistics and evidence + +For each agent and primary metric, report: + +- all per-seed values; +- mean; +- normal-approximation 95% confidence interval across seeds. + +The PR workflow emits `artifacts/sym-arch-001/report.json` and uploads it as a GitHub Actions artifact. + +Wall-clock runtime is recorded as observational evidence only. The reproducible resource anchors are: + +- representation dimension; +- number of training/update observations per agent; +- SSM state width. + +No energy or parameter-count superiority claim is licensed by this first tranche. + +## Claims this experiment can license + +At most: + +> Under the preregistered SYM-ARCH-001 synthetic continual-relational task family, the tested HDC-LTC+Hebbian mechanism did/did not show a practically meaningful advantage over the included controls. + +It cannot by itself license: + +- "Symthaea beats transformers"; +- "Symthaea has solved continual learning"; +- "Symthaea has demonstrated AGI"; +- "Symthaea is more compute-efficient than frontier AI"; +- a causal reasoning claim stronger than contingency adaptation. + +## Follow-up if informative + +A positive or mixed result should be followed by a separate preregistered tranche with stronger external baselines (small GRU/LSTM, trainable SSM, small transformer where practical), explicit compute/memory accounting, larger held-out relational families, and a genuine intervention-based causal adaptation task. + +A null or negative result should be treated as architecture guidance: identify which ablation fails, preserve the evidence, and change the mechanism rather than loosening this experiment's criteria. From 377dca06de0057539761157dd8a8de4ab74dc76a Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Mon, 24 Aug 2026 23:07:43 +0200 Subject: [PATCH 05/12] ci(research): run SYM-ARCH-001 on pull requests --- .github/workflows/sym-arch-001.yml | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/workflows/sym-arch-001.yml diff --git a/.github/workflows/sym-arch-001.yml b/.github/workflows/sym-arch-001.yml new file mode 100644 index 0000000000..fc75666a0f --- /dev/null +++ b/.github/workflows/sym-arch-001.yml @@ -0,0 +1,114 @@ +name: SYM-ARCH-001 Architecture Discrimination + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/sym-arch-001.yml' + - 'crates/domains/symthaea-psych-bench/**' + - 'crates/domains/symthaea-ssm/**' + - 'crates/core/symthaea-core/src/hdc/**' + - 'docs/research/SYM_ARCH_001_PREREGISTRATION_2026-08-24.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + sym-arch-001: + name: Preregistered continual/compositional campaign + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + with: + components: rustfmt, clippy + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-001-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-001- + + - name: Format check for experiment files + run: | + cargo fmt --all -- --check + + - name: Compile and run experiment unit tests + run: | + cargo test -p symthaea-psych-bench --lib benchmarks::architecture -- --nocapture + + - name: Run preregistered SYM-ARCH-001 campaign + run: | + cargo run -p symthaea-psych-bench --bin sym_arch_001 --release -- \ + --output artifacts/sym-arch-001/report.json + + - name: Validate and summarize evidence + run: | + python3 - <<'PY' + import json + from pathlib import Path + + path = Path('artifacts/sym-arch-001/report.json') + report = json.loads(path.read_text()) + assert report['schema'] == 'symthaea.sym-arch-001.v1' + assert report['decision']['verdict'] in {'PASS', 'MIXED', 'NULL', 'NEGATIVE'} + assert len(report['agents']) == 5 + assert all(len(a['per_seed']) == 16 for a in report['agents']) + + print('=== SYM-ARCH-001 ===') + print('verdict:', report['decision']['verdict']) + for a in report['agents']: + print( + f"{a['agent']:20s} " + f"retention={a['retention']['mean']:.3f} " + f"composition={a['compositional']['mean']:.3f} " + f"forgetting={a['forgetting']['mean']:.3f} " + f"reversal={a['reversal_final']['mean']:.3f} " + f"latency={a['reversal_latency']['mean']:.1f}" + ) + + summary = Path(__import__('os').environ['GITHUB_STEP_SUMMARY']) + with summary.open('a') as f: + f.write('## SYM-ARCH-001 result\n\n') + f.write(f"**Verdict:** `{report['decision']['verdict']}`\n\n") + f.write('| agent | retention | composition | forgetting ↓ | reversal | latency ↓ |\n') + f.write('|---|---:|---:|---:|---:|---:|\n') + for a in report['agents']: + f.write( + f"| `{a['agent']}` | {a['retention']['mean']:.3f} | " + f"{a['compositional']['mean']:.3f} | {a['forgetting']['mean']:.3f} | " + f"{a['reversal_final']['mean']:.3f} | {a['reversal_latency']['mean']:.1f} |\n" + ) + d = report['decision'] + f.write('\nCandidate deltas vs strongest control: ') + f.write( + f"retention `{d['retention_delta_vs_best_control']:+.3f}`, " + f"composition `{d['compositional_delta_vs_best_control']:+.3f}`, " + f"reversal `{d['reversal_delta_vs_best_control']:+.3f}`, " + f"forgetting `{d['forgetting_delta_vs_best_control']:+.3f}`.\n" + ) + PY + sha256sum artifacts/sym-arch-001/report.json | tee artifacts/sym-arch-001/SHA256SUMS + + - name: Upload evidence artifact + uses: actions/upload-artifact@v4 + with: + name: sym-arch-001-${{ github.sha }} + path: artifacts/sym-arch-001/ + retention-days: 90 From e7406955fa3797cc8293df6af6fbd5113978b7f9 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Mon, 24 Aug 2026 23:13:09 +0200 Subject: [PATCH 06/12] ci(research): keep formatting separate from evidence execution --- .github/workflows/sym-arch-001.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sym-arch-001.yml b/.github/workflows/sym-arch-001.yml index fc75666a0f..eee2bfaf52 100644 --- a/.github/workflows/sym-arch-001.yml +++ b/.github/workflows/sym-arch-001.yml @@ -31,8 +31,6 @@ jobs: - name: Setup Rust uses: dtolnay/rust-toolchain@1.96.0 - with: - components: rustfmt, clippy - name: Cache cargo uses: actions/cache@v4 @@ -45,10 +43,9 @@ jobs: restore-keys: | ${{ runner.os }}-sym-arch-001- - - name: Format check for experiment files - run: | - cargo fmt --all -- --check - + # Formatting remains the responsibility of normal repository CI. Keep + # the research workflow focused on whether the preregistered experiment + # compiles, executes, and emits valid evidence. - name: Compile and run experiment unit tests run: | cargo test -p symthaea-psych-bench --lib benchmarks::architecture -- --nocapture From eb931fbaed960f066406caf969c5aaad350c9d64 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 10:12:14 +0200 Subject: [PATCH 07/12] fix(test): remove stale loop trial fixture field --- .../symthaea-psych-bench/src/harness/neuromod_correlation.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs index b4c849fdc3..c6056aad3d 100644 --- a/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs +++ b/crates/domains/symthaea-psych-bench/src/harness/neuromod_correlation.rs @@ -241,11 +241,6 @@ mod tests { cycle_time_us: 1000 + (i as u64 * 50), learning_occurred: i % 5 == 0, reward: 0.0, - // Added 2026-07-31 to unblock the crate: another session's - // LoopTrialResult gained `cycle_reward` and this test fixture was - // collateral damage. Synthetic data, so 0.0 matches the - // neighbouring `reward`/`moral_score` fixtures. - cycle_reward: 0.0, oxytocin: 0.3, moral_score: 0.0, bath_entropy: 0.5, From 0351cc5abc74293b1f2d6f2c6cbc5ff0862dfc8a Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 11:56:55 +0200 Subject: [PATCH 08/12] docs(research): freeze SYM-ARCH-001 negative result --- .../SYM_ARCH_001_RESULT_2026-08-25.md | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 docs/research/SYM_ARCH_001_RESULT_2026-08-25.md diff --git a/docs/research/SYM_ARCH_001_RESULT_2026-08-25.md b/docs/research/SYM_ARCH_001_RESULT_2026-08-25.md new file mode 100644 index 0000000000..95544e7716 --- /dev/null +++ b/docs/research/SYM_ARCH_001_RESULT_2026-08-25.md @@ -0,0 +1,227 @@ +# SYM-ARCH-001 Result — 2026-08-25 + +**Status:** frozen result record +**Preregistered verdict:** **NEGATIVE** +**Experiment PR:** #53 +**Merged to `main`:** `6a25ea6d5faccabff2351d577a808d2cb36eb35b` + +## Executive result + +SYM-ARCH-001 produced its first valid confirmatory observation on 2026-08-25. + +Under the decision rule frozen before behavioral results were observed, the benchmark-local `hdc_ltc_hebbian` candidate received a **NEGATIVE** verdict. + +The candidate did not beat the strongest control by the preregistered `0.05` margin on any of the three target metrics. It regressed beyond the `0.05` tolerance on all three: + +- retention delta vs strongest control: `-0.296875`; +- held-out compositional delta vs strongest control: `-0.25390625`; +- reversal-final delta vs strongest control: `-0.486328125`; +- target wins at margin: `0`; +- target regressions beyond tolerance: `3`. + +Mean forgetting for the candidate was `0.04296875`, compared with best-control forgetting `0.02604167`. This difference remained inside the frozen forgetting tolerance and was **not** the condition that caused the negative verdict. + +The correct conclusion is narrow: + +> The tested 512-dimensional benchmark-local HDC-LTC-Hebbian bundle did not outperform the preregistered controls on the joint retention/composition/reversal criterion under the frozen SYM-ARCH-001 protocol. + +This is not evidence that HDC, LTC/CfC, Hebbian plasticity, or Symthaea as a whole are ineffective. + +## Evidence provenance + +The first hosted attempt failed before behavioral execution because an unrelated psych-bench test fixture still supplied the removed `LoopTrialResult::cycle_reward` field. No behavioral result from that attempt was observed or used to alter the experiment. + +The only repair before the valid retry removed that stale synthetic fixture field/comment: + +- frozen scientific head before repair: `e7406955fa3797cc8293df6af6fbd5113978b7f9`; +- retry PR head: `eb931fbaed960f066406caf969c5aaad350c9d64`; +- repair delta: one unrelated file, `0` additions / `5` deletions; +- no changes to the benchmark implementation, preregistration, thresholds, seeds, task splits, timing, agents, or decision rule. + +The valid run was GitHub Actions run `32825418156`, attempt/run number 3. Every experiment step completed successfully: + +1. experiment unit tests; +2. release campaign execution; +3. report-schema/evidence validation; +4. SHA-256 generation; +5. artifact upload. + +GitHub Actions evaluated synthetic pull-request merge commit: + +`60165a666eb6e5a358445acf3f087a7775e38f0b` + +whose commit message is: + +`Merge eb931fbaed960f066406caf969c5aaad350c9d64 into 8de0ca10e69c2da42844fd7a202e639bf21e32bc` + +A commit comparison from PR head `eb931fba...` to tested merge commit `60165a...` contains **zero file differences**. Thus the tested working tree is equivalent to the retry PR head. + +### Artifact integrity + +Uploaded artifact: + +- artifact id: `9557446155`; +- artifact name: `sym-arch-001-60165a666eb6e5a358445acf3f087a7775e38f0b`; +- artifact ZIP SHA-256 reported by GitHub: `7f8424e2285f0f020b2bed07f7b716caafa10e547e3a3bf1837e307c7fdc4684`; +- `report.json` SHA-256: `841b78ffdd8e63b801d54c667f12ff6ab4db00abe88ec1403c74544500901985`. + +The downloaded `SHA256SUMS` entry matched an independent recomputation of `report.json`. + +## Frozen configuration + +| Parameter | Value | +|---|---:| +| representation dimension | 512 | +| independent RNG seeds in v1 | 16 | +| train epochs per world | 16 | +| reversal epochs | 12 | +| prototype alpha | 0.15 | +| Hebbian learning rate | 0.002 | +| target win margin | 0.05 | +| regression tolerance | 0.05 | +| worlds | 4 | +| categorical values per factor | 4 | +| held-out combinations per world | 4 | +| observations per agent | 960 | + +The world identity is explicitly encoded in the input. HDC-LTC evolution uses fixed `dt = 0.05`. Reversal is a contingency inversion, not a causal-intervention benchmark. + +## Agent results + +The intervals below are the normal-approximation 95% intervals emitted by the frozen v1 implementation. They are retained exactly as part of the result record; they are not upgraded post hoc to the stronger paired/hierarchical statistics designed for SYM-ARCH-002. + +| Agent | Retention | Composition | Forgetting ↓ | Reversal final | Reversal latency ↓ | +|---|---:|---:|---:|---:|---:| +| `linear_sgd` | 0.9284 | 0.8008 | 0.0260 | 0.8086 | 175.8 | +| `vanilla_hdc` | 0.6120 | 0.6445 | 0.3307 | **0.9961** | **48.1** | +| `fixed_diagonal_ssm` | 0.5990 | 0.5938 | 0.2305 | 0.7617 | 54.3 | +| `hdc_ltc_frozen` | 0.6445 | 0.6133 | 0.0443 | 0.4219 | 184.1 | +| `hdc_ltc_hebbian` | 0.6315 | 0.5469 | 0.0430 | 0.5098 | 170.4 | + +### Emitted 95% intervals + +| Agent | Retention 95% CI | Composition 95% CI | Forgetting 95% CI | Reversal 95% CI | Latency 95% CI | +|---|---|---|---|---|---| +| `linear_sgd` | [0.9179, 0.9389] | [0.7707, 0.8308] | [0.0158, 0.0363] | [0.7206, 0.8966] | [168.2, 183.4] | +| `vanilla_hdc` | [0.5778, 0.6461] | [0.5714, 0.7176] | [0.2937, 0.3678] | [0.9909, 1.0013] | [46.2, 50.0] | +| `fixed_diagonal_ssm` | [0.5829, 0.6150] | [0.5621, 0.6254] | [0.2059, 0.2550] | [0.7302, 0.7933] | [46.7, 61.8] | +| `hdc_ltc_frozen` | [0.5910, 0.6981] | [0.5498, 0.6768] | [0.0272, 0.0613] | [0.3369, 0.5069] | [166.5, 201.6] | +| `hdc_ltc_hebbian` | [0.5784, 0.6846] | [0.4904, 0.6033] | [0.0269, 0.0590] | [0.4209, 0.5987] | [145.6, 195.3] | + +The `vanilla_hdc` normal interval extending slightly above `1.0` is a useful limitation of v1's unconstrained normal approximation. The result is left unchanged. SYM-ARCH-002 uses stronger paired/hierarchical methods rather than retroactively changing v1 statistics. + +## What the negative result actually teaches us + +### 1. The bundled HDC-LTC-Hebbian candidate is not justified by this task + +The candidate lost badly to the strongest observed controls on all three target metrics. There is no basis for carrying the entire 001 bundle forward as a favored architecture and tuning it until it wins. + +Future work should factorize the mechanisms and make each one earn its complexity independently. + +### 2. `linear_sgd` is a serious control, not a ceremonial baseline + +`linear_sgd` dominated final retention and held-out composition in this task family: + +- retention `0.9284`; +- composition `0.8008`; +- forgetting `0.0260`. + +That is a warning against assuming a cognitively elaborate architecture must beat a simple online learner on a small explicitly contextualized relational task. + +SYM-ARCH-002B therefore begins with stronger simple controls before adding more sophisticated neural/SSM models. + +### 3. Vanilla HDC shows a striking stability/plasticity tradeoff + +`vanilla_hdc` had weak retention and severe forgetting but almost perfect final reversal performance: + +- retention `0.6120`; +- forgetting `0.3307`; +- reversal final `0.9961`; +- reversal latency `48.1`. + +This does not prove a general HDC property because the result includes the benchmark-local prototype update/readout. It does show that v1 contains qualitatively different behavioral regimes that a single aggregate score would hide. + +### 4. Low forgetting is not sufficient evidence of good continual learning + +Both liquid variants showed low mean forgetting (`~0.043–0.044`) while their absolute retention/composition remained modest. + +That exposes an important ambiguity: + +> A system can appear stable because it retains knowledge, or because it never acquired the task strongly enough for a large peak-to-final decline to occur. + +This is precisely why SYM-ARCH-002 adds the full task-by-time performance matrix, acquisition speed, average incremental accuracy, forward/backward transfer, and learning-curve measures. The need for those metrics is now empirically motivated by 001 rather than merely methodological preference. + +## Strictly post-hoc mechanism observation + +The following comparison was **not** the preregistered primary decision and must not be reported as confirmatory evidence. + +Comparing `hdc_ltc_hebbian` against its closest frozen-liquid ablation, `hdc_ltc_frozen`, gives mean directional differences across the same 16 seeds: + +| Metric | Hebbian − frozen | +|---|---:| +| retention | -0.0130 | +| composition | -0.0664 | +| forgetting | -0.0013 | +| reversal final | +0.0879 | +| reversal latency | -13.6 trials | + +Directionally, Hebbian adaptation improved reversal behavior while composition worsened. No paired preregistered inference was defined for this secondary contrast, so this is an **exploratory hypothesis generator only**. + +A valid follow-up is not to tune the existing candidate. It is to preregister a mechanism-factorized comparison where representation, temporal dynamics, plasticity, and readout are controlled separately. + +## Limitations frozen with the result + +SYM-ARCH-001 is a mechanism-level synthetic benchmark, not a full Symthaea or frontier-model evaluation. + +Important limitations include: + +- only four hand-authored relation rules; +- 16 RNG seeds do not represent 16 independent task programs; +- explicit world/context identity in every encoded input; +- fixed `dt = 0.05`, so the central irregular-time motivation of CfC/LTC is not exercised; +- only four held-out combinations per world; +- deterministic held-out selection; +- prototype readout is not matched against every representation/control; +- strongest control is selected separately by metric; +- normal-approximation intervals rather than paired hierarchical inference; +- no prospective power calculation; +- no task-free drift; +- no resource-matched neural baseline; +- reversal is contingency inversion rather than causal intervention; +- the old live reward-consumption path is intentionally bypassed. + +These are reasons to design a stronger next experiment, not reasons to reinterpret the frozen result. + +## Consequence for SYM-ARCH-002 + +The result strengthens the rationale for the 002 program already tracked in issue #55: + +1. preserve DEV / CONFIRM / REPL separation; +2. generate genuinely different environments rather than treating RNG seeds as independent cognitive tasks; +3. measure full acquisition/retention dynamics; +4. use paired environment-level inference and prospective power; +5. validate generated benchmarks before scoring architectures; +6. attack benchmarks with shortcut controls; +7. compare fixed random features, HDC, and stronger conventional learners using matched readouts/resources; +8. isolate HDC representation, liquid dynamics, adaptive timescales, Hebbian plasticity, and associative memory separately; +9. test latent/task-free context and irregular physical time where liquid dynamics should have a theoretically specific advantage; +10. reserve causal claims for intervention-based mechanism tests and independent replication. + +The purpose of 002 is therefore not to rescue the 001 candidate. It is to discover which, if any, Symthaea mechanisms earn their complexity under harder controls. + +## Claim ceiling + +This document supports the statement: + +> SYM-ARCH-001 produced a valid preregistered **NEGATIVE** result for the tested benchmark-local HDC-LTC-Hebbian candidate under the frozen 2026-08-24 protocol. + +It does not support claims that: + +- Symthaea as a whole failed; +- HDC is inferior in general; +- LTC/CfC is ineffective; +- Hebbian plasticity is harmful in general; +- linear SGD is a superior general cognitive architecture; +- vanilla HDC is generally optimal for reversal learning. + +Those questions require the mechanism-specific, task-diverse, resource-aware follow-up program. \ No newline at end of file From 76a5904b596ab2bd877d5638b61969034e617423 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 14:56:19 +0200 Subject: [PATCH 09/12] ci(research): add guarded NixOS runner fallback --- .github/workflows/sym-arch-002a-core.yml | 46 ++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sym-arch-002a-core.yml b/.github/workflows/sym-arch-002a-core.yml index b2ec04a6fc..4fcf677345 100644 --- a/.github/workflows/sym-arch-002a-core.yml +++ b/.github/workflows/sym-arch-002a-core.yml @@ -14,6 +14,12 @@ on: - 'Cargo.lock' workflow_dispatch: +concurrency: + # Keep only the latest automatic run for a branch. Manual dispatch remains + # non-cancellable because it is a deliberate request for a definitive run. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + permissions: contents: read @@ -22,8 +28,11 @@ env: RUST_BACKTRACE: 1 jobs: - experimental-core: - name: Validate experiment infrastructure + experimental-core-hosted: + name: Validate experiment infrastructure (hosted) + # Fork PRs must never execute untrusted code on the repository's self-hosted + # NixOS runner. Manual dispatch also keeps the standard hosted path. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest timeout-minutes: 20 @@ -60,3 +69,36 @@ jobs: - name: Check psych-bench library run: | cargo check -p symthaea-psych-bench --lib + + experimental-core-nixos: + name: Validate experiment infrastructure + # Security boundary: only branches inside this repository may execute on the + # trusted self-hosted runner. Fork PRs are routed to the hosted job above. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: [self-hosted, gpu, nixos] + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - name: Verify pinned Rust toolchain + run: | + set -euo pipefail + nix develop --command rustc --version | grep -F 'rustc 1.96.0 ' + nix develop --command cargo --version + nix develop --command rustfmt --version + + - name: Check changed Rust formatting + run: | + nix develop --command rustfmt --edition 2024 --check \ + crates/domains/symthaea-psych-bench/src/experiment/mod.rs \ + crates/domains/symthaea-psych-bench/src/experiment/confirmatory.rs \ + crates/domains/symthaea-psych-bench/src/lib.rs + + - name: Run experimental-core tests + run: | + nix develop --command cargo test -p symthaea-psych-bench --lib experiment -- --nocapture + + - name: Check psych-bench library + run: | + nix develop --command cargo check -p symthaea-psych-bench --lib From d66c4eea96d94a9fc36c43d087067241674c6a71 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 14:58:13 +0200 Subject: [PATCH 10/12] ci(research): keep hosted core and supersede stale runs --- .github/workflows/sym-arch-002a-core.yml | 40 ++---------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/.github/workflows/sym-arch-002a-core.yml b/.github/workflows/sym-arch-002a-core.yml index 4fcf677345..c48b655bfb 100644 --- a/.github/workflows/sym-arch-002a-core.yml +++ b/.github/workflows/sym-arch-002a-core.yml @@ -28,11 +28,8 @@ env: RUST_BACKTRACE: 1 jobs: - experimental-core-hosted: - name: Validate experiment infrastructure (hosted) - # Fork PRs must never execute untrusted code on the repository's self-hosted - # NixOS runner. Manual dispatch also keeps the standard hosted path. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + experimental-core: + name: Validate experiment infrastructure runs-on: ubuntu-latest timeout-minutes: 20 @@ -69,36 +66,3 @@ jobs: - name: Check psych-bench library run: | cargo check -p symthaea-psych-bench --lib - - experimental-core-nixos: - name: Validate experiment infrastructure - # Security boundary: only branches inside this repository may execute on the - # trusted self-hosted runner. Fork PRs are routed to the hosted job above. - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - runs-on: [self-hosted, gpu, nixos] - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v4 - - - name: Verify pinned Rust toolchain - run: | - set -euo pipefail - nix develop --command rustc --version | grep -F 'rustc 1.96.0 ' - nix develop --command cargo --version - nix develop --command rustfmt --version - - - name: Check changed Rust formatting - run: | - nix develop --command rustfmt --edition 2024 --check \ - crates/domains/symthaea-psych-bench/src/experiment/mod.rs \ - crates/domains/symthaea-psych-bench/src/experiment/confirmatory.rs \ - crates/domains/symthaea-psych-bench/src/lib.rs - - - name: Run experimental-core tests - run: | - nix develop --command cargo test -p symthaea-psych-bench --lib experiment -- --nocapture - - - name: Check psych-bench library - run: | - nix develop --command cargo check -p symthaea-psych-bench --lib From d2331611288cb5bc99a3d30fcd9dc9581b2b287d Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 15:05:49 +0200 Subject: [PATCH 11/12] ci(research): freeze SYM-ARCH-001 post-result execution --- .github/workflows/sym-arch-001.yml | 67 +++++++++++++++++------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/.github/workflows/sym-arch-001.yml b/.github/workflows/sym-arch-001.yml index eee2bfaf52..d3547a0546 100644 --- a/.github/workflows/sym-arch-001.yml +++ b/.github/workflows/sym-arch-001.yml @@ -1,3 +1,9 @@ +# SYM-ARCH-001 concluded on 2026-08-25 with a frozen NEGATIVE verdict. +# +# Ordinary pull requests now perform only implementation regression checks on the +# exact historical experiment surface. The full campaign is intentionally manual: +# a later reproduction must not be mistaken for a second confirmatory sample or +# overwrite the durable result in docs/research/SYM_ARCH_001_RESULT_2026-08-25.md. name: SYM-ARCH-001 Architecture Discrimination on: @@ -5,14 +11,20 @@ on: branches: [main] paths: - '.github/workflows/sym-arch-001.yml' - - 'crates/domains/symthaea-psych-bench/**' - - 'crates/domains/symthaea-ssm/**' - - 'crates/core/symthaea-core/src/hdc/**' + - 'crates/domains/symthaea-psych-bench/src/benchmarks/architecture.rs' + - 'crates/domains/symthaea-psych-bench/src/benchmarks/mod.rs' + - 'crates/domains/symthaea-psych-bench/src/bin/sym_arch_001.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' - 'docs/research/SYM_ARCH_001_PREREGISTRATION_2026-08-24.md' - - 'Cargo.toml' - - 'Cargo.lock' workflow_dispatch: +concurrency: + # Automatic branch runs supersede stale queued/running checks. Manual historical + # reproductions are deliberate evidence requests and therefore never cancel one + # another or get cancelled by a subsequent push. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + permissions: contents: read @@ -22,7 +34,7 @@ env: jobs: sym-arch-001: - name: Preregistered continual/compositional campaign + name: Historical experiment regression / reproduction runs-on: ubuntu-latest timeout-minutes: 30 @@ -43,33 +55,34 @@ jobs: restore-keys: | ${{ runner.os }}-sym-arch-001- - # Formatting remains the responsibility of normal repository CI. Keep - # the research workflow focused on whether the preregistered experiment - # compiles, executes, and emits valid evidence. - name: Compile and run experiment unit tests run: | cargo test -p symthaea-psych-bench --lib benchmarks::architecture -- --nocapture + cargo check -p symthaea-psych-bench --bin sym_arch_001 - - name: Run preregistered SYM-ARCH-001 campaign + - name: Run historical SYM-ARCH-001 reproduction + if: github.event_name == 'workflow_dispatch' run: | cargo run -p symthaea-psych-bench --bin sym_arch_001 --release -- \ - --output artifacts/sym-arch-001/report.json + --output artifacts/sym-arch-001-reproduction/report.json - - name: Validate and summarize evidence + - name: Validate and summarize reproduction evidence + if: github.event_name == 'workflow_dispatch' run: | python3 - <<'PY' import json from pathlib import Path - path = Path('artifacts/sym-arch-001/report.json') + path = Path('artifacts/sym-arch-001-reproduction/report.json') report = json.loads(path.read_text()) assert report['schema'] == 'symthaea.sym-arch-001.v1' assert report['decision']['verdict'] in {'PASS', 'MIXED', 'NULL', 'NEGATIVE'} assert len(report['agents']) == 5 assert all(len(a['per_seed']) == 16 for a in report['agents']) - print('=== SYM-ARCH-001 ===') - print('verdict:', report['decision']['verdict']) + print('=== SYM-ARCH-001 historical reproduction ===') + print('reproduction verdict:', report['decision']['verdict']) + print('frozen scientific verdict remains: NEGATIVE (2026-08-25)') for a in report['agents']: print( f"{a['agent']:20s} " @@ -82,8 +95,10 @@ jobs: summary = Path(__import__('os').environ['GITHUB_STEP_SUMMARY']) with summary.open('a') as f: - f.write('## SYM-ARCH-001 result\n\n') - f.write(f"**Verdict:** `{report['decision']['verdict']}`\n\n") + f.write('## SYM-ARCH-001 historical reproduction\n\n') + f.write('This run is a **reproducibility/regression check**, not a new confirmatory sample.\n\n') + f.write('The frozen scientific verdict remains **`NEGATIVE` (2026-08-25)** and is recorded in `docs/research/SYM_ARCH_001_RESULT_2026-08-25.md`.\n\n') + f.write(f"**Reproduction verdict:** `{report['decision']['verdict']}`\n\n") f.write('| agent | retention | composition | forgetting ↓ | reversal | latency ↓ |\n') f.write('|---|---:|---:|---:|---:|---:|\n') for a in report['agents']: @@ -92,20 +107,14 @@ jobs: f"{a['compositional']['mean']:.3f} | {a['forgetting']['mean']:.3f} | " f"{a['reversal_final']['mean']:.3f} | {a['reversal_latency']['mean']:.1f} |\n" ) - d = report['decision'] - f.write('\nCandidate deltas vs strongest control: ') - f.write( - f"retention `{d['retention_delta_vs_best_control']:+.3f}`, " - f"composition `{d['compositional_delta_vs_best_control']:+.3f}`, " - f"reversal `{d['reversal_delta_vs_best_control']:+.3f}`, " - f"forgetting `{d['forgetting_delta_vs_best_control']:+.3f}`.\n" - ) PY - sha256sum artifacts/sym-arch-001/report.json | tee artifacts/sym-arch-001/SHA256SUMS + sha256sum artifacts/sym-arch-001-reproduction/report.json \ + | tee artifacts/sym-arch-001-reproduction/SHA256SUMS - - name: Upload evidence artifact + - name: Upload historical reproduction artifact + if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v4 with: - name: sym-arch-001-${{ github.sha }} - path: artifacts/sym-arch-001/ + name: sym-arch-001-reproduction-${{ github.sha }} + path: artifacts/sym-arch-001-reproduction/ retention-days: 90 From b43ddb1ce6e0060b7adf288810b2faecb437ec51 Mon Sep 17 00:00:00 2001 From: Tristan Stoltz Date: Tue, 25 Aug 2026 15:09:34 +0200 Subject: [PATCH 12/12] ci: supersede stale showroom integrity runs --- .github/workflows/showroom-integrity.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/showroom-integrity.yml b/.github/workflows/showroom-integrity.yml index 4a44977a55..dfa557e557 100644 --- a/.github/workflows/showroom-integrity.yml +++ b/.github/workflows/showroom-integrity.yml @@ -6,6 +6,12 @@ on: pull_request: branches: [main] +concurrency: + # Showroom integrity is repo-wide, but only the latest commit for a branch/PR + # needs to consume runner capacity. Supersede stale automatic validations. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: validate-ring0: name: Ring 0 Buildability Audit @@ -13,10 +19,10 @@ jobs: if: github.repository == 'Luminous-Dynamics/symthaea' steps: - uses: actions/checkout@v4 - + - name: Install Rust uses: dtolnay/rust-toolchain@stable - + - name: Verify Metadata run: cargo metadata --no-deps --format-version 1 @@ -27,6 +33,6 @@ jobs: run: | # Ensure no forbidden dependencies ! rg -i 'mycelix|prism|holochain|swarm|robotics|symtropy-robotics|zero.?knowledge|zkp|pqc|private monorepo|internal only' . - + # Check license head -n 5 LICENSE | grep "GNU AFFERO GENERAL PUBLIC LICENSE"