diff --git a/.github/workflows/sym-arch-002a6-measurement-dynamics.yml b/.github/workflows/sym-arch-002a6-measurement-dynamics.yml new file mode 100644 index 000000000..94d400719 --- /dev/null +++ b/.github/workflows/sym-arch-002a6-measurement-dynamics.yml @@ -0,0 +1,73 @@ +name: SYM-ARCH-002A6 Measurement Dynamics + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: + - main + - research/sym-arch-002a-core-v1 + paths: + - '.github/workflows/sym-arch-002a6-measurement-dynamics.yml' + - 'crates/domains/symthaea-psych-bench/src/experiment/measurement.rs' + - 'crates/domains/symthaea-psych-bench/src/experiment/measurement_validity.rs' + - 'crates/domains/symthaea-psych-bench/src/lib.rs' + - 'crates/domains/symthaea-psych-bench/Cargo.toml' + - 'docs/research/SYM_ARCH_002A6_MEASUREMENT_DYNAMICS_V1.md' + - 'Cargo.toml' + - 'Cargo.lock' + workflow_dispatch: + +concurrency: + # Automatic branch runs supersede stale checks. Deliberate manual validation + # runs are unique and therefore never cancel one another. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + measurement-dynamics: + name: Validate acquisition and resource measurement core + # Draft stacked PRs cannot merge and should not consume scarce runner capacity. + # Promoting to ready-for-review explicitly triggers this exact gate. + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@1.96.0 + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-sym-arch-002a6-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sym-arch-002a6- + + - name: Check formatting + run: | + cargo fmt --all -- --check + + - name: Run A6 measurement tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_measurement:: -- --nocapture + + - name: Run A6 criterion-validity tests + run: | + cargo test -p symthaea-psych-bench --lib experiment_measurement_validity:: -- --nocapture + + - name: Check psych-bench library + run: | + cargo check -p symthaea-psych-bench --lib diff --git a/crates/domains/symthaea-psych-bench/src/experiment/measurement.rs b/crates/domains/symthaea-psych-bench/src/experiment/measurement.rs new file mode 100644 index 000000000..cd3ac2ec4 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/measurement.rs @@ -0,0 +1,541 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Online acquisition/resource measurement primitives for SYM-ARCH-002A6. +//! +//! This module is architecture-agnostic. It records prequential correctness +//! before each update, inference/update latency for the same step, and post-update +//! resource state. It intentionally does not decide whether any model "wins". + +use serde::{Deserialize, Serialize}; + +pub const ONLINE_MEASUREMENT_TRACE_SCHEMA_V1: &str = "symthaea.online-measurement-trace/v1"; +const TRACE_HASH_DOMAIN: &[u8] = b"symthaea.online-measurement-trace.hash/v1"; + +fn canonical_hash(domain: &[u8], value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?; + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(&[0]); + hasher.update(&bytes); + Ok(hasher.finalize().to_hex().to_string()) +} + +fn looks_like_digest(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct LearningCriterion { + /// Rolling prequential window size in examples. + pub window_size: usize, + /// Required rolling accuracy in [0,1]. + pub accuracy_threshold: f64, + /// Number of consecutive qualifying windows required before criterion is met. + pub consecutive_windows: usize, +} + +impl LearningCriterion { + pub fn validate(&self) -> Result<(), String> { + if self.window_size == 0 { + return Err("learning criterion window_size must be positive".into()); + } + if self.consecutive_windows == 0 { + return Err("learning criterion consecutive_windows must be positive".into()); + } + if !self.accuracy_threshold.is_finite() + || self.accuracy_threshold < 0.0 + || self.accuracy_threshold > 1.0 + { + return Err("learning criterion accuracy_threshold must be finite in [0,1]".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResourceSnapshot { + /// Number of trainable scalar parameters after the update. + pub trainable_parameters: usize, + /// Total persistent model state retained across examples. + pub persistent_state_bytes: u64, + /// Portion of persistent state attributable to replay/examples, when present. + pub replay_bytes: u64, + /// Portion of persistent state attributable to temporal/recurrent state. + pub temporal_state_bytes: u64, + /// Process resident-set size sampled after the update, when available. + pub rss_bytes: Option, +} + +impl ResourceSnapshot { + pub fn validate(&self) -> Result<(), String> { + if self.replay_bytes > self.persistent_state_bytes { + return Err("replay bytes cannot exceed total persistent state bytes".into()); + } + if self.temporal_state_bytes > self.persistent_state_bytes { + return Err("temporal state bytes cannot exceed total persistent state bytes".into()); + } + Ok(()) + } +} + +/// One prequential online-learning step. +/// +/// `correct_before_update` must reflect a prediction made before the label is +/// consumed by the learner. Latencies are observational measurements and must not +/// be used as hidden tuning signals in CONFIRM/REPL streams. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnlineStepMeasurement { + pub correct_before_update: bool, + pub inference_latency_ns: u64, + pub update_latency_ns: u64, + pub resource_after_update: ResourceSnapshot, +} + +impl OnlineStepMeasurement { + pub fn validate(&self) -> Result<(), String> { + self.resource_after_update.validate() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OnlineMeasurementTrace { + pub schema: String, + /// Digest of the experiment manifest controlling this trace. + pub manifest_digest: String, + /// Digest of the exact task program represented by this contiguous phase. + pub task_program_digest: String, + /// Digest of separately recorded runtime/hardware/compiler context. Latency + /// comparisons across different runtime contexts require an explicit policy. + pub runtime_context_digest: String, + /// Human-readable contiguous phase identity, e.g. `world-b-acquisition`. + pub phase_id: String, + /// Optional digest of the exact learner/spec state before the first step. + pub initial_state_digest: Option, + pub steps: Vec, +} + +impl OnlineMeasurementTrace { + pub fn validate(&self) -> Result<(), String> { + if self.schema != ONLINE_MEASUREMENT_TRACE_SCHEMA_V1 { + return Err(format!("unsupported online-measurement schema: {}", self.schema)); + } + for (name, digest) in [ + ("manifest", &self.manifest_digest), + ("task_program", &self.task_program_digest), + ("runtime_context", &self.runtime_context_digest), + ] { + if !looks_like_digest(digest) { + return Err(format!( + "measurement trace {name}_digest must be a 32-byte hex digest" + )); + } + } + if self.phase_id.trim().is_empty() { + return Err("measurement trace phase_id must be non-empty".into()); + } + if let Some(digest) = &self.initial_state_digest { + if !looks_like_digest(digest) { + return Err("measurement trace initial_state_digest must be a 32-byte hex digest".into()); + } + } + if self.steps.is_empty() { + return Err("online measurement trace must contain at least one step".into()); + } + for step in &self.steps { + step.validate()?; + } + Ok(()) + } + + pub fn digest(&self) -> Result { + self.validate()?; + canonical_hash(TRACE_HASH_DOMAIN, self) + } + + pub fn correctness(&self) -> Vec { + self.steps + .iter() + .map(|step| step.correct_before_update) + .collect() + } + + /// Number of examples observed when the frozen rolling criterion is first met. + /// Returns `None` if the criterion is never sustained. + pub fn examples_to_criterion( + &self, + criterion: &LearningCriterion, + ) -> Result, String> { + self.validate()?; + criterion.validate()?; + if self.steps.len() < criterion.window_size { + return Ok(None); + } + + let mut prefix = Vec::with_capacity(self.steps.len() + 1); + prefix.push(0usize); + for step in &self.steps { + let next = prefix.last().copied().unwrap_or(0) + step.correct_before_update as usize; + prefix.push(next); + } + + let mut consecutive = 0usize; + for end in criterion.window_size..=self.steps.len() { + let start = end - criterion.window_size; + let correct = prefix[end] - prefix[start]; + let accuracy = correct as f64 / criterion.window_size as f64; + if accuracy + 1e-12 >= criterion.accuracy_threshold { + consecutive += 1; + if consecutive >= criterion.consecutive_windows { + return Ok(Some(end)); + } + } else { + consecutive = 0; + } + } + Ok(None) + } + + /// Mean cumulative prequential accuracy across learning steps. + /// + /// This is a normalized right-rectangle area under the cumulative-accuracy + /// learning curve. Two runs with the same overall accuracy can differ here + /// when one acquires useful behavior earlier. + pub fn cumulative_accuracy_auc(&self) -> Result { + self.validate()?; + let mut correct = 0usize; + let mut area = 0.0; + for (index, step) in self.steps.iter().enumerate() { + correct += step.correct_before_update as usize; + area += correct as f64 / (index + 1) as f64; + } + Ok(area / self.steps.len() as f64) + } + + pub fn overall_prequential_accuracy(&self) -> Result { + self.validate()?; + let correct = self + .steps + .iter() + .filter(|step| step.correct_before_update) + .count(); + Ok(correct as f64 / self.steps.len() as f64) + } + + /// Accuracy over the final `window_size` pre-update predictions. + pub fn terminal_window_accuracy(&self, window_size: usize) -> Result, String> { + self.validate()?; + if window_size == 0 { + return Err("terminal accuracy window_size must be positive".into()); + } + if self.steps.len() < window_size { + return Ok(None); + } + let correct = self.steps[self.steps.len() - window_size..] + .iter() + .filter(|step| step.correct_before_update) + .count(); + Ok(Some(correct as f64 / window_size as f64)) + } + + pub fn summarize( + &self, + criterion: &LearningCriterion, + ) -> Result { + self.validate()?; + criterion.validate()?; + let inference: Vec = self.steps.iter().map(|step| step.inference_latency_ns).collect(); + let update: Vec = self.steps.iter().map(|step| step.update_latency_ns).collect(); + let resources: Vec = self + .steps + .iter() + .map(|step| step.resource_after_update.clone()) + .collect(); + Ok(OnlineMeasurementSummary { + observations: self.steps.len(), + examples_to_criterion: self.examples_to_criterion(criterion)?, + overall_prequential_accuracy: self.overall_prequential_accuracy()?, + terminal_window_accuracy: self.terminal_window_accuracy(criterion.window_size)?, + cumulative_accuracy_auc: self.cumulative_accuracy_auc()?, + inference_latency: LatencySummary::from_samples(&inference)?, + update_latency: LatencySummary::from_samples(&update)?, + resources: ResourceTraceSummary::from_snapshots(&resources)?, + trace_digest: self.digest()?, + }) + } +} + +fn percentile_linear(sorted: &[u64], probability: f64) -> f64 { + debug_assert!(!sorted.is_empty()); + let position = probability.clamp(0.0, 1.0) * (sorted.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + sorted[lower] as f64 + } else { + let weight = position - lower as f64; + sorted[lower] as f64 * (1.0 - weight) + sorted[upper] as f64 * weight + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LatencySummary { + pub samples: usize, + pub total_ns: u64, + pub mean_ns: f64, + pub p50_ns: f64, + pub p95_ns: f64, + /// None only when every measured latency sample is zero. + pub throughput_per_second: Option, +} + +impl LatencySummary { + pub fn from_samples(samples: &[u64]) -> Result { + if samples.is_empty() { + return Err("latency summary requires at least one sample".into()); + } + let total_ns = samples.iter().try_fold(0u64, |total, sample| { + total + .checked_add(*sample) + .ok_or_else(|| "latency total overflow".to_string()) + })?; + let mut sorted = samples.to_vec(); + sorted.sort_unstable(); + let throughput_per_second = if total_ns == 0 { + None + } else { + Some(samples.len() as f64 * 1_000_000_000.0 / total_ns as f64) + }; + Ok(Self { + samples: samples.len(), + total_ns, + mean_ns: total_ns as f64 / samples.len() as f64, + p50_ns: percentile_linear(&sorted, 0.50), + p95_ns: percentile_linear(&sorted, 0.95), + throughput_per_second, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResourceTraceSummary { + pub samples: usize, + pub final_trainable_parameters: usize, + pub peak_trainable_parameters: usize, + pub final_persistent_state_bytes: u64, + pub peak_persistent_state_bytes: u64, + pub peak_replay_bytes: u64, + pub peak_temporal_state_bytes: u64, + pub peak_rss_bytes: Option, +} + +impl ResourceTraceSummary { + pub fn from_snapshots(snapshots: &[ResourceSnapshot]) -> Result { + if snapshots.is_empty() { + return Err("resource summary requires at least one snapshot".into()); + } + for snapshot in snapshots { + snapshot.validate()?; + } + let last = snapshots.last().expect("non-empty snapshots"); + Ok(Self { + samples: snapshots.len(), + final_trainable_parameters: last.trainable_parameters, + peak_trainable_parameters: snapshots + .iter() + .map(|snapshot| snapshot.trainable_parameters) + .max() + .unwrap_or(0), + final_persistent_state_bytes: last.persistent_state_bytes, + peak_persistent_state_bytes: snapshots + .iter() + .map(|snapshot| snapshot.persistent_state_bytes) + .max() + .unwrap_or(0), + peak_replay_bytes: snapshots + .iter() + .map(|snapshot| snapshot.replay_bytes) + .max() + .unwrap_or(0), + peak_temporal_state_bytes: snapshots + .iter() + .map(|snapshot| snapshot.temporal_state_bytes) + .max() + .unwrap_or(0), + peak_rss_bytes: snapshots.iter().filter_map(|snapshot| snapshot.rss_bytes).max(), + }) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OnlineMeasurementSummary { + pub observations: usize, + pub examples_to_criterion: Option, + pub overall_prequential_accuracy: f64, + pub terminal_window_accuracy: Option, + pub cumulative_accuracy_auc: f64, + pub inference_latency: LatencySummary, + pub update_latency: LatencySummary, + pub resources: ResourceTraceSummary, + pub trace_digest: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digest(byte: &str) -> String { + byte.repeat(64) + } + + fn resource(persistent: u64, replay: u64, temporal: u64, rss: Option) -> ResourceSnapshot { + ResourceSnapshot { + trainable_parameters: 10, + persistent_state_bytes: persistent, + replay_bytes: replay, + temporal_state_bytes: temporal, + rss_bytes: rss, + } + } + + fn trace(correctness: &[bool]) -> OnlineMeasurementTrace { + OnlineMeasurementTrace { + schema: ONLINE_MEASUREMENT_TRACE_SCHEMA_V1.into(), + manifest_digest: digest("a"), + task_program_digest: digest("b"), + runtime_context_digest: digest("c"), + phase_id: "phase-a".into(), + initial_state_digest: Some(digest("d")), + steps: correctness + .iter() + .enumerate() + .map(|(index, correct)| OnlineStepMeasurement { + correct_before_update: *correct, + inference_latency_ns: 10 + index as u64, + update_latency_ns: 20 + index as u64, + resource_after_update: resource( + 1_000 + index as u64, + 100, + 50, + Some(5_000 + index as u64), + ), + }) + .collect(), + } + } + + #[test] + fn criterion_requires_sustained_rolling_accuracy() { + let trace = trace(&[false, false, true, true, true, true]); + let criterion = LearningCriterion { + window_size: 3, + accuracy_threshold: 2.0 / 3.0, + consecutive_windows: 2, + }; + assert_eq!(trace.examples_to_criterion(&criterion).unwrap(), Some(5)); + } + + #[test] + fn criterion_returns_none_when_not_reached() { + let trace = trace(&[false, true, false, true, false, true]); + let criterion = LearningCriterion { + window_size: 3, + accuracy_threshold: 1.0, + consecutive_windows: 2, + }; + assert_eq!(trace.examples_to_criterion(&criterion).unwrap(), None); + } + + #[test] + fn cumulative_auc_rewards_earlier_acquisition_at_same_overall_accuracy() { + let early = trace(&[true, true, false, false]); + let late = trace(&[false, false, true, true]); + assert_eq!(early.overall_prequential_accuracy().unwrap(), 0.5); + assert_eq!(late.overall_prequential_accuracy().unwrap(), 0.5); + assert!(early.cumulative_accuracy_auc().unwrap() > late.cumulative_accuracy_auc().unwrap()); + } + + #[test] + fn terminal_window_accuracy_is_not_overall_accuracy() { + let trace = trace(&[false, false, true, true]); + assert_eq!(trace.overall_prequential_accuracy().unwrap(), 0.5); + assert_eq!(trace.terminal_window_accuracy(2).unwrap(), Some(1.0)); + } + + #[test] + fn latency_summary_reports_deterministic_percentiles_and_throughput() { + let summary = LatencySummary::from_samples(&[10, 20, 30, 40]).unwrap(); + assert_eq!(summary.samples, 4); + assert_eq!(summary.total_ns, 100); + assert!((summary.mean_ns - 25.0).abs() < 1e-12); + assert!((summary.p50_ns - 25.0).abs() < 1e-12); + assert!((summary.p95_ns - 38.5).abs() < 1e-12); + assert!((summary.throughput_per_second.unwrap() - 40_000_000.0).abs() < 1e-6); + } + + #[test] + fn zero_latency_does_not_invent_infinite_throughput() { + let summary = LatencySummary::from_samples(&[0, 0, 0]).unwrap(); + assert_eq!(summary.throughput_per_second, None); + } + + #[test] + fn resource_summary_tracks_final_and_peak_state() { + let snapshots = vec![ + resource(100, 10, 20, Some(1_000)), + resource(140, 30, 25, Some(1_200)), + resource(120, 20, 22, None), + ]; + let summary = ResourceTraceSummary::from_snapshots(&snapshots).unwrap(); + assert_eq!(summary.final_persistent_state_bytes, 120); + assert_eq!(summary.peak_persistent_state_bytes, 140); + assert_eq!(summary.peak_replay_bytes, 30); + assert_eq!(summary.peak_temporal_state_bytes, 25); + assert_eq!(summary.peak_rss_bytes, Some(1_200)); + } + + #[test] + fn trace_digest_is_order_task_runtime_and_manifest_bound() { + let first = trace(&[true, false, true]); + let reordered = trace(&[false, true, true]); + assert_ne!(first.digest().unwrap(), reordered.digest().unwrap()); + + let mut changed = first.clone(); + changed.manifest_digest = digest("e"); + assert_ne!(first.digest().unwrap(), changed.digest().unwrap()); + changed = first.clone(); + changed.task_program_digest = digest("e"); + assert_ne!(first.digest().unwrap(), changed.digest().unwrap()); + changed = first.clone(); + changed.runtime_context_digest = digest("e"); + assert_ne!(first.digest().unwrap(), changed.digest().unwrap()); + changed = first.clone(); + changed.phase_id = "phase-b".into(); + assert_ne!(first.digest().unwrap(), changed.digest().unwrap()); + } + + #[test] + fn invalid_resource_component_fails_closed() { + let mut invalid = trace(&[true]); + invalid.steps[0].resource_after_update.replay_bytes = 2_000; + assert!(invalid.validate().is_err()); + } + + #[test] + fn summary_binds_learning_latency_resource_and_trace_identity() { + let trace = trace(&[false, true, true, true]); + let criterion = LearningCriterion { + window_size: 2, + accuracy_threshold: 1.0, + consecutive_windows: 2, + }; + let summary = trace.summarize(&criterion).unwrap(); + assert_eq!(summary.observations, 4); + assert_eq!(summary.examples_to_criterion, Some(4)); + assert_eq!(summary.terminal_window_accuracy, Some(1.0)); + assert!(looks_like_digest(&summary.trace_digest)); + assert_eq!(summary.inference_latency.samples, 4); + assert_eq!(summary.update_latency.samples, 4); + assert_eq!(summary.resources.samples, 4); + } +} diff --git a/crates/domains/symthaea-psych-bench/src/experiment/measurement_validity.rs b/crates/domains/symthaea-psych-bench/src/experiment/measurement_validity.rs new file mode 100644 index 000000000..af5d45564 --- /dev/null +++ b/crates/domains/symthaea-psych-bench/src/experiment/measurement_validity.rs @@ -0,0 +1,182 @@ +// Copyright (C) 2024-2026 Tristan Stoltz / Luminous Dynamics +// SPDX-License-Identifier: AGPL-3.0-or-later +// Commercial licensing: see COMMERCIAL_LICENSE.md at repository root +//! Construct-validity guard for SYM-ARCH-002A6 acquisition criteria. +//! +//! A learning threshold is not claim-bearing if a known chance/majority/shortcut +//! reference can already meet it. This audit makes that boundary explicit while +//! leaving inferential uncertainty to A2 and shortcut discovery to A4. + +use crate::experiment_measurement::LearningCriterion; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AcquisitionCriterionStatus { + Admissible, + ReferenceConfounded, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AcquisitionCriterionAudit { + pub criterion: LearningCriterion, + /// Frozen accuracy ceiling of the preregistered simple reference/control. + pub reference_accuracy_ceiling: f64, + /// Minimum practical accuracy excess required above the reference ceiling. + pub minimum_excess_over_reference: f64, + /// Integer number correct actually needed in one rolling window. + pub required_correct_per_window: usize, + /// Finite-window accuracy represented by `required_correct_per_window`. + pub effective_accuracy_threshold: f64, + pub effective_excess_over_reference: f64, + pub status: AcquisitionCriterionStatus, + pub qualifiers: Vec, +} + +fn validate_probability(name: &str, value: f64) -> Result<(), String> { + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + return Err(format!("{name} must be finite in [0,1]")); + } + Ok(()) +} + +/// Audit a frozen acquisition criterion against a preregistered reference ceiling. +/// +/// This is a construct-validity guard, not a significance test. An admissible +/// result still requires A2 uncertainty/power analysis and A4 benchmark controls. +pub fn audit_acquisition_criterion( + criterion: LearningCriterion, + reference_accuracy_ceiling: f64, + minimum_excess_over_reference: f64, +) -> Result { + criterion.validate()?; + validate_probability("reference_accuracy_ceiling", reference_accuracy_ceiling)?; + validate_probability( + "minimum_excess_over_reference", + minimum_excess_over_reference, + )?; + + let raw_required = criterion.accuracy_threshold * criterion.window_size as f64; + let required_correct_per_window = (raw_required - 1e-12) + .ceil() + .clamp(0.0, criterion.window_size as f64) as usize; + let effective_accuracy_threshold = + required_correct_per_window as f64 / criterion.window_size as f64; + let effective_excess_over_reference = + effective_accuracy_threshold - reference_accuracy_ceiling; + + let mut qualifiers = Vec::new(); + if effective_accuracy_threshold <= reference_accuracy_ceiling + 1e-12 { + qualifiers.push(format!( + "effective criterion {:.6} does not exceed frozen reference ceiling {:.6}", + effective_accuracy_threshold, reference_accuracy_ceiling + )); + } + if effective_excess_over_reference + 1e-12 < minimum_excess_over_reference { + qualifiers.push(format!( + "effective excess {:.6} is below required practical margin {:.6}", + effective_excess_over_reference, minimum_excess_over_reference + )); + } + + Ok(AcquisitionCriterionAudit { + criterion, + reference_accuracy_ceiling, + minimum_excess_over_reference, + required_correct_per_window, + effective_accuracy_threshold, + effective_excess_over_reference, + status: if qualifiers.is_empty() { + AcquisitionCriterionStatus::Admissible + } else { + AcquisitionCriterionStatus::ReferenceConfounded + }, + qualifiers, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn criterion_above_reference_and_margin_is_admissible() { + let audit = audit_acquisition_criterion( + LearningCriterion { + window_size: 20, + accuracy_threshold: 0.80, + consecutive_windows: 2, + }, + 0.50, + 0.10, + ) + .unwrap(); + assert_eq!(audit.required_correct_per_window, 16); + assert!((audit.effective_accuracy_threshold - 0.80).abs() < 1e-12); + assert_eq!(audit.status, AcquisitionCriterionStatus::Admissible); + assert!(audit.qualifiers.is_empty()); + } + + #[test] + fn shortcut_level_threshold_is_reference_confounded() { + let audit = audit_acquisition_criterion( + LearningCriterion { + window_size: 20, + accuracy_threshold: 0.60, + consecutive_windows: 3, + }, + 0.65, + 0.05, + ) + .unwrap(); + assert_eq!(audit.status, AcquisitionCriterionStatus::ReferenceConfounded); + assert!(!audit.qualifiers.is_empty()); + } + + #[test] + fn finite_window_resolution_is_reported_not_hidden() { + let audit = audit_acquisition_criterion( + LearningCriterion { + window_size: 4, + accuracy_threshold: 0.81, + consecutive_windows: 1, + }, + 0.50, + 0.10, + ) + .unwrap(); + assert_eq!(audit.required_correct_per_window, 4); + assert_eq!(audit.effective_accuracy_threshold, 1.0); + assert_eq!(audit.status, AcquisitionCriterionStatus::Admissible); + } + + #[test] + fn practical_margin_can_fail_even_when_reference_is_exceeded() { + let audit = audit_acquisition_criterion( + LearningCriterion { + window_size: 20, + accuracy_threshold: 0.70, + consecutive_windows: 2, + }, + 0.65, + 0.10, + ) + .unwrap(); + assert_eq!(audit.status, AcquisitionCriterionStatus::ReferenceConfounded); + assert!(audit + .qualifiers + .iter() + .any(|qualifier| qualifier.contains("practical margin"))); + } + + #[test] + fn audit_rejects_invalid_reference_contract() { + let criterion = LearningCriterion { + window_size: 20, + accuracy_threshold: 0.80, + consecutive_windows: 2, + }; + assert!(audit_acquisition_criterion(criterion, 1.1, 0.05).is_err()); + assert!(audit_acquisition_criterion(criterion, 0.5, -0.01).is_err()); + } +} diff --git a/crates/domains/symthaea-psych-bench/src/lib.rs b/crates/domains/symthaea-psych-bench/src/lib.rs index df0a8b5ea..7198372f4 100644 --- a/crates/domains/symthaea-psych-bench/src/lib.rs +++ b/crates/domains/symthaea-psych-bench/src/lib.rs @@ -56,6 +56,10 @@ pub mod benchmarks; pub mod experiment; #[path = "experiment/confirmatory.rs"] pub mod experiment_confirmatory; +#[path = "experiment/measurement.rs"] +pub mod experiment_measurement; +#[path = "experiment/measurement_validity.rs"] +pub mod experiment_measurement_validity; pub mod harness; pub mod substrate_transfer; pub mod wm; diff --git a/docs/research/SYM_ARCH_002A6_MEASUREMENT_DYNAMICS_V1.md b/docs/research/SYM_ARCH_002A6_MEASUREMENT_DYNAMICS_V1.md new file mode 100644 index 000000000..2b9bab53f --- /dev/null +++ b/docs/research/SYM_ARCH_002A6_MEASUREMENT_DYNAMICS_V1.md @@ -0,0 +1,238 @@ +# SYM-ARCH-002A6 — Online Measurement Dynamics v1 + +**Status:** measurement-infrastructure tranche; no architecture result + +**Tracks:** #55 + +**Base:** SYM-ARCH-002A experimental core (#57) + +## Why this exists + +SYM-ARCH-001 exposed a stability/plasticity ambiguity: low measured forgetting can coexist with weak absolute acquisition. A learner that never acquires a task strongly may appear stable simply because there is little useful behavior to forget. + +A6 therefore measures the *trajectory of learning itself*, not only final task scores. + +It is architecture-agnostic and carries no Symthaea performance claim. + +## Prequential measurement rule + +Every online-learning step is ordered as: + +1. present learner-visible input; +2. produce prediction **before consuming the current label**; +3. record correctness and inference latency; +4. apply the labeled update; +5. record update latency; +6. record post-update resource state. + +Post-update evaluation on the same item must not be substituted for `correct_before_update` in claim-bearing acquisition traces. + +This is the core anti-leakage rule for acquisition metrics. + +## Trace provenance + +Every `OnlineMeasurementTrace` is bound to: + +- exact experiment-manifest digest; +- exact task-program digest; +- separately recorded runtime-context digest; +- human-readable contiguous `phase_id`; +- optional exact initial learner/spec-state digest; +- every ordered online step. + +The trace itself receives a domain-separated BLAKE3 digest. + +A continual run should normally emit a separate trace for each acquisition/adaptation phase rather than collapsing all task transitions into one global learning curve. + +Examples: + +- `world-a-acquisition`; +- `world-b-acquisition`; +- `post-reversal-recovery`; +- `drift-episode-03`. + +## Acquisition metrics + +### Examples to criterion + +The criterion is frozen before claim-bearing evaluation: + +- rolling-window size; +- accuracy threshold; +- number of consecutive qualifying windows. + +The metric returns the number of examples observed when the sustained criterion is first reached, or `None` when it is never reached. + +Requiring consecutive qualifying windows prevents one lucky local burst from being called acquisition. + +The criterion must not be chosen after inspecting CONFIRM/REPL learning curves. + +### Criterion admissibility against simple references + +A threshold can be implemented perfectly and still be scientifically weak if a known chance, majority, marginal, or other preregistered shortcut reference can already satisfy it. + +`experiment_measurement_validity::audit_acquisition_criterion` therefore freezes: + +- the learning criterion; +- a preregistered reference-accuracy ceiling; +- the minimum practical excess required above that ceiling. + +The audit also exposes the criterion's **finite-window resolution**. For a window of `n`, a nominal threshold is converted into the integer number correct actually required and the corresponding effective accuracy threshold. + +It returns: + +- `admissible` when the effective threshold clears the frozen reference ceiling by at least the required practical margin; +- `reference_confounded` otherwise, with explicit qualifiers. + +This is a construct-validity guard, not a significance test. Passing it does not replace: + +- A4's shortcut-control campaign; +- A2's paired/hierarchical uncertainty; +- prospective power; +- multiple-comparison control. + +A claim-bearing examples-to-criterion result must not use a `reference_confounded` criterion. + +### Overall prequential accuracy + +Accuracy across every pre-update prediction in the phase. + +This is an average over the entire learning history and is **not** described as final/terminal performance. + +### Terminal-window accuracy + +Accuracy over the final frozen criterion-window length. This separates end-of-phase performance from the average learning history. + +### Cumulative-accuracy AUC + +A6 defines normalized cumulative-accuracy AUC as the mean cumulative prequential accuracy across steps (a right-rectangle integral of the cumulative-accuracy curve). + +It lies in `[0,1]` and rewards earlier acquisition. Two traces can have identical overall accuracy but different cumulative AUC when one learns useful behavior earlier. + +This definition is frozen for A6; it must not later be silently replaced by ordinary mean correctness, post-update accuracy, or a different smoothing procedure. + +## Latency and throughput + +Each step records: + +- inference latency in nanoseconds; +- update latency in nanoseconds. + +Each latency series reports: + +- sample count; +- total time; +- mean; +- p50; +- p95; +- observations/second when total measured time is nonzero. + +Percentiles use deterministic linear interpolation over sorted observed nanosecond samples. + +### Runtime-context boundary + +Latency numbers are not hardware-independent model properties. + +The trace therefore requires a `runtime_context_digest` representing separately frozen runtime information such as relevant CPU/GPU, operating environment, compiler/build profile, affinity/threading policy, and measurement protocol. + +A latency superiority claim across different runtime-context digests requires an explicit cross-runtime comparison policy; A6 itself does not authorize one. + +## Resource trace + +After each update A6 records: + +- trainable scalar parameters; +- total persistent state bytes; +- replay bytes; +- temporal/recurrent state bytes; +- optional process RSS. + +The summary reports: + +- final and peak trainable parameter counts; +- final and peak persistent state; +- peak replay state; +- peak temporal state; +- peak observed RSS. + +Replay/temporal byte counts are components of persistent state and may not individually exceed the reported persistent-state total. + +RSS is observational process-level memory, not a substitute for model-state accounting. + +## What must be frozen before CONFIRM + +For every claim-bearing acquisition comparison freeze at least: + +- experiment manifest; +- task-program identity; +- phase boundaries; +- prequential ordering rule; +- learning criterion; +- reference-accuracy ceiling used to audit the criterion; +- minimum practical excess above that reference; +- observation budget; +- evaluation cadence; +- latency timing protocol; +- runtime-context schema; +- resource-accounting semantics; +- primary acquisition metric; +- comparator and SESOI; +- statistical analysis from A2. + +DEV may be used to choose these values. CONFIRM/REPL may not tune them. + +## Relationship to the R matrix + +A6 does not replace `R[t_train][t_eval]`. + +The two answer different questions: + +- R matrix: what is retained/transferred across tasks? +- A6 trace: how quickly and efficiently is behavior acquired or recovered inside a phase? + +A system with low forgetting but poor acquisition should therefore be visible as: + +- apparently favorable retention/forgetting summaries; +- slow or absent examples-to-criterion; +- poor cumulative-accuracy AUC; +- weak terminal-window accuracy. + +That combination must not be described as strong continual learning. + +## Acceptance tests + +The exact PR head must demonstrate: + +1. invalid trace provenance fails closed; +2. sustained rolling criterion is required; +3. criterion miss returns `None` rather than an invented latency; +4. earlier acquisition raises cumulative AUC at matched overall accuracy; +5. terminal-window accuracy is distinct from overall prequential accuracy; +6. latency p50/p95 and throughput summaries are deterministic; +7. zero measured latency cannot create infinite throughput; +8. resource peaks/finals are reported correctly; +9. impossible replay/persistent-state accounting fails closed; +10. trace digest changes with order, manifest, task, runtime context, or phase; +11. summary binds acquisition, latency, resource, and trace identity; +12. criterion audit distinguishes admissible from reference-confounded thresholds; +13. finite-window criterion resolution is explicit; +14. the full psych-bench library compiles. + +## Claim ceiling + +Merging A6 supports only: + +> Symthaea psych-bench contains provenance-bound prequential acquisition, criterion-validity, latency/throughput, and resource measurement primitives suitable for later preregistered continual-learning experiments. + +It does **not** support: + +- a Symthaea performance claim; +- a claim that any mechanism learns faster; +- a hardware-independent latency claim; +- a resource-efficiency claim across unmatched budgets; +- a claim that low forgetting implies successful learning; +- a claim-bearing acquisition threshold when the criterion audit is `reference_confounded`. + +## Next use + +Once A-series infrastructure is executable and green, use A6 traces in DEV with B1 and later baselines to identify sensible frozen acquisition criteria and estimate effect/variance structure before opening CONFIRM.