From 94f22be16a6aa49160e4c9dd956cc1e208a7b3b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Seidel?= Date: Sat, 31 Oct 2026 09:33:38 +0100 Subject: [PATCH] Replay entry point, cache hit through replay, profile-aware replay tests (#790, #788, #787) --- .github/workflows/ci.yml | 6 + CHANGELOG.md | 1 + docs/OUTPUT_CONTRACT.md | 3 +- src/audit/artifacts.rs | 287 ++++++++++++++++-- src/audit/mod.rs | 5 +- src/audit/support_matrix.rs | 37 ++- src/cli/args.rs | 48 +++ src/cli/commands.rs | 2 + src/cli/runners.rs | 144 +++++---- src/main.rs | 20 +- .../architecture/unchecked_normalize.txt | 2 +- tests/replay_paths_test.rs | 278 +++++++++++++++++ 12 files changed, 736 insertions(+), 97 deletions(-) create mode 100644 tests/replay_paths_test.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a719627..ec04fe98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -297,6 +297,12 @@ jobs: CI: "true" run: cargo test --test integration_test -- --ignored --nocapture --test-threads=1 + - name: Run replay-path tests (fresh vs cache vs replay) + env: + AUDITMYSITE_BROWSER: ${{ steps.setup-chrome.outputs.chrome-path }} + CI: "true" + run: cargo test --test replay_paths_test -- --ignored --nocapture --test-threads=1 + # Schema/vulnerable-libs/easy-language detection-corpus tests # (#553-#558; the WCAG corpus has its own job below): real Chrome, diffed against ground-truth fixtures. # These are separate `#[ignore]`-gated test binaries and were never diff --git a/CHANGELOG.md b/CHANGELOG.md index 866f4792..d35a0c2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and verification for each change is in the ### Added +- **Replay:** `auditmysite replay ` evaluates a stored snapshot without browser or network and reports it with `execution.environment.source: replay` and the verdict as exit code. `--session`, `--state-id` and `--storage-state` given to it are assertions about the stored measurement profile: a snapshot measured under another one is refused (`ReplayError::OtherMeasurement`, naming field, stored and asserted value) instead of being shown as the same measurement; `--dns-check`, `--check-ssr-content`, `--isolate-third-party-impact` and `--ai-transparency` are refused, because a replay shows the checks its run recorded. `--reuse-cache` answers through the same replay (`source: cache`); an unreadable, incomplete, version-mismatched or differently configured entry is a miss that prints its reason (`CacheMiss`), and the lossy reconstruction is no longer used for cache hits. `tests/replay_paths_test.rs` (Chrome, local fixture) checks that fresh run, cache hit and replay give one canonical model and that a single-page app whose content changes after a client-side navigation is explained by its measurement profile (`audit::profile_differences`). - **Snapshot:** `snapshot.json` now holds the whole collected report, the document identity (requested/final URL, status, redirects), the run language and the versions it was written under (snapshot schema, fingerprint, collection contract, measurement profile, evaluation, decision contract, rule-set digest). `audit::replay` evaluates it without browser or network into the same model as the fresh run; `output::format_json_replay` exports it with `detail.module_detail: full`. Incomplete or version-mismatched snapshots are reported as such (`ReplayError`) and are a cache miss, never read under current semantics. Cache entries of earlier builds are not reused. Details: [Snapshot, replay and validation paths](https://github.com/casoon/auditmysite/blob/main/docs/OUTPUT_CONTRACT.md#snapshot-replay-and-validation-paths). - **Validation:** `tests/validation_path_matrix.rs` checks that every fallible export entry refuses contradictory data and that unchecked `normalize()` calls in `cli`, `output` and `studio` do not grow. - **Navigation:** `navigation.redirect_chain` lists the main document's redirects (URL and status, in order) next to `requested_url`, `final_url` and `redirect_count`. diff --git a/docs/OUTPUT_CONTRACT.md b/docs/OUTPUT_CONTRACT.md index 63205ab1..f0ab0e30 100644 --- a/docs/OUTPUT_CONTRACT.md +++ b/docs/OUTPUT_CONTRACT.md @@ -130,7 +130,8 @@ The cache entry's `snapshot.json` is the collected evidence, independent of any - `versions` records the snapshot schema, the content fingerprint, the collection contract, the measurement-profile version, the evaluation version (normalization, scoring, decision derivation), the decision contract and a digest of the shared rule catalogue. A snapshot is replayed only when all of them equal the current values. - `audit::replay(snapshot)` evaluates a snapshot without browser or network and returns the same `NormalizedReport` as the fresh run. Only `timestamp`, `duration_ms` and `execution.environment.source` (`replay`) may differ. `output::format_json_replay` exports it with `detail.module_detail: full`. -- A snapshot that lacks parts (`ReplayError::Incomplete`, naming them) or was written under other versions (`ReplayError::Incompatible`, naming field, stored and current value) is never evaluated under today's semantics. `load_artifacts` treats it as a cache miss. +- A snapshot that lacks parts (`ReplayError::Incomplete`, naming them) or was written under other versions (`ReplayError::Incompatible`, naming field, stored and current value) is never evaluated under today's semantics. `load_artifacts` treats it as a cache miss; `lookup_artifacts` and `cached_report` return the reason (`CacheMiss`: absent, unreadable, other configuration, not replayable), which the CLI prints. +- `--reuse-cache` answers through `replay_report_as(snapshot, "cache")`, `auditmysite replay ` through `replay_report`. `SnapshotArtifact::check_measurement(&ExpectedProfile)` refuses a snapshot whose session policy, state id or storage-state digest differs from the asserted one (`ReplayError::OtherMeasurement`); `audit::profile_differences` names the differences between two profiles. A replay measures nothing, so the extra-check flags are refused for it (support matrix, mode `Replay`). - `format_json_cached` works on a stored `NormalizedReport` alone and says so with `module_detail: not_restored`; an empty module block there never stands for restored data. - Bump `SNAPSHOT_SCHEMA_VERSION` when the shape of the collected report changes in a way old data would survive deserialization, and `EVALUATION_VERSION` when a change moves a score, finding or verdict for unchanged input. diff --git a/src/audit/artifacts.rs b/src/audit/artifacts.rs index 0f030019..87469b4e 100644 --- a/src/audit/artifacts.rs +++ b/src/audit/artifacts.rs @@ -31,7 +31,7 @@ use crate::audit::normalized::NormalizedReport; use crate::audit::PerformanceResults; use crate::audit::{AccessibilityScorer, AuditReport}; use crate::cli::WcagLevel; -use crate::error::Result; +use crate::error::{AuditError, Result}; use crate::mobile::MobileFriendliness; use crate::security::SecurityAnalysis; use crate::seo::SeoAnalysis; @@ -243,6 +243,10 @@ pub enum ReplayError { /// Written under other versions than the current ones. #[error("snapshot is incompatible: {}", format_mismatches(.0))] Incompatible(Vec), + /// Measured under another profile than the one the caller asserts: the + /// replay would present another measurement as the requested one. + #[error("snapshot was measured under another profile: {}", format_mismatches(.0))] + OtherMeasurement(Vec), /// The replayed data contradicts itself. #[error("replayed data violates the report contract: {0}")] Contract(#[from] crate::audit::ReportContractError), @@ -338,12 +342,21 @@ impl SnapshotArtifact { /// The report a replay evaluates: the collected data with the in-memory /// layers rebuilt from the persisted tree. The provenance says `replay`. pub fn replay_report(snapshot: &SnapshotArtifact) -> std::result::Result { + replay_report_as(snapshot, "replay") +} + +/// [`replay_report`] with the provenance a caller names: `replay` for an +/// explicit replay, `cache` when a cache hit is answered through it. +pub fn replay_report_as( + snapshot: &SnapshotArtifact, + source: &str, +) -> std::result::Result { snapshot.replayability()?; let mut report = snapshot .collected .clone() .ok_or(ReplayError::Incomplete(vec!["collected"]))?; - report.accessibility.execution.environment.source = "replay".to_string(); + report.accessibility.execution.environment.source = source.to_string(); report.screen_reader_audit = Some(rebuild_screen_reader( &report.url, report.timestamp, @@ -362,6 +375,112 @@ pub fn replay(snapshot: &SnapshotArtifact) -> std::result::Result, + pub state_id: Option, + /// SHA-256 of the declared storage state. + pub storage_state_sha256: Option, +} + +/// How two measurement profiles differ, as `(field, a, b)` in the +/// `stored`/`current` slots. Equal profiles differ in nothing; a page whose +/// content differs between two runs of one URL is explained by the profile +/// only if this list is not empty. +pub fn profile_differences( + a: &crate::audit::MeasurementProfile, + b: &crate::audit::MeasurementProfile, +) -> Vec { + let mut out = Vec::new(); + let mut check = |field: &'static str, a: String, b: String| { + if a != b { + out.push(VersionMismatch { + field, + stored: a, + current: b, + }); + } + }; + let sha = |p: &crate::audit::MeasurementProfile| { + p.storage_state + .as_ref() + .map_or("none".to_string(), |s| s.sha256.clone()) + }; + check( + "session_policy", + a.session_policy.clone(), + b.session_policy.clone(), + ); + check( + "state_id", + a.state_id.clone().unwrap_or_else(|| "none".to_string()), + b.state_id.clone().unwrap_or_else(|| "none".to_string()), + ); + check("storage_state", sha(a), sha(b)); + out +} + +impl SnapshotArtifact { + /// Refuses a snapshot measured under another profile than the asserted + /// one. Only the asserted fields are compared. + pub fn check_measurement( + &self, + expected: &ExpectedProfile, + ) -> std::result::Result<(), ReplayError> { + self.replayability()?; + let stored = self + .collected + .as_ref() + .map(|c| c.accessibility.execution.scope.measurement_profile.clone()) + .unwrap_or_default(); + let mut wanted = stored.clone(); + if let Some(session) = &expected.session_policy { + wanted.session_policy = session.clone(); + } + if let Some(id) = &expected.state_id { + wanted.state_id = Some(id.clone()); + } + if let Some(sha) = &expected.storage_state_sha256 { + wanted.storage_state = Some(crate::audit::StorageStateRef { + sha256: sha.clone(), + cookies: 0, + origins: 0, + }); + } + let differences = profile_differences(&stored, &wanted); + if differences.is_empty() { + Ok(()) + } else { + Err(ReplayError::OtherMeasurement(differences)) + } + } + + /// Reads a stored snapshot from a `snapshot.json` or the cache entry + /// directory holding one. + pub fn read(path: &std::path::Path) -> Result { + let file = if path.is_dir() { + path.join("snapshot.json") + } else { + path.to_path_buf() + }; + let bytes = fs::read(&file).map_err(|e| AuditError::FileError { + path: file.clone(), + reason: e.to_string(), + })?; + serde_json::from_slice(&bytes).map_err(|e| AuditError::FileError { + path: file, + reason: format!("not a snapshot: {e}"), + }) + } + + /// The snapshot of the current cache entry for `url`. + pub fn read_cached(url: &str) -> Result { + Self::read(&artifact_dir(url)?) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AuditArtifacts { pub fetch: FetchArtifact, @@ -423,13 +542,69 @@ fn write_entry(dir: &std::path::Path, wcag_level: &str, artifacts: &AuditArtifac Ok(()) } +/// Why the cache cannot answer a request. +#[derive(Debug, thiserror::Error)] +pub enum CacheMiss { + /// Nothing stored for this URL (or the entry is not complete on disk). + #[error("no cached audit for this URL")] + Absent, + /// The entry exists but cannot be read. + #[error("cached audit {} is unreadable ({reason})", .path.display())] + Unreadable { path: PathBuf, reason: String }, + /// The entry was produced for another audit scope or run configuration. + #[error("cached audit was produced with a different audit configuration")] + OtherConfiguration, + /// The snapshot cannot be replayed under the current semantics. + #[error("cached audit is not reusable: {0}")] + NotReplayable(#[from] ReplayError), +} + +/// A cache entry, or why there is none. +#[derive(Debug)] +pub enum CacheLookup { + Hit(Box), + Miss(CacheMiss), +} + pub fn load_artifacts(url: &str) -> Result> { - read_entry(&artifact_dir(url)?) + Ok(match lookup_artifacts(url)? { + CacheLookup::Hit(artifacts) => Some(*artifacts), + CacheLookup::Miss(CacheMiss::Absent) => None, + CacheLookup::Miss(reason) => { + warn!("{reason}; running fresh"); + None + } + }) +} + +/// Looks up the cache entry of `url`, naming the reason when there is none. +pub fn lookup_artifacts(url: &str) -> Result { + Ok(read_entry(&artifact_dir(url)?)) +} + +/// The report a `--reuse-cache` hit answers with: the stored snapshot +/// replayed (#790), never a reconstruction. An entry written for another +/// audit configuration, or one whose snapshot is incomplete or from other +/// versions, is a miss with its reason — never the current semantics applied +/// to old data. +pub fn cached_report( + url: &str, + expected_signature: &str, +) -> Result> { + let cached = match lookup_artifacts(url)? { + CacheLookup::Hit(cached) => cached, + CacheLookup::Miss(reason) => return Ok(Err(reason)), + }; + if !cache_matches_signature(&cached.meta, expected_signature) { + return Ok(Err(CacheMiss::OtherConfiguration)); + } + Ok(replay_report_as(&cached.snapshot, "cache").map_err(CacheMiss::from)) } -fn read_entry(dir: &std::path::Path) -> Result> { +fn read_entry(dir: &std::path::Path) -> CacheLookup { + use CacheLookup::{Hit, Miss}; if !dir.exists() { - return Ok(None); + return Miss(CacheMiss::Absent); } let fetch_path = dir.join("fetch.json"); @@ -439,12 +614,12 @@ fn read_entry(dir: &std::path::Path) -> Result> { let meta_path = dir.join("meta.json"); if !fetch_path.exists() || !snapshot_path.exists() || !audit_path.exists() { - return Ok(None); + return Miss(CacheMiss::Absent); } - // A truncated or otherwise corrupt cache entry must not abort the run — treat - // any read/parse failure as a cache miss so the caller falls back to a fresh - // audit (#405). + // A truncated or otherwise corrupt cache entry must not abort the run — it + // is a cache miss with a reason, so the caller falls back to a fresh audit + // (#405). macro_rules! load_or_miss { ($path:expr, $ty:ty) => { match fs::read(&$path) @@ -452,13 +627,11 @@ fn read_entry(dir: &std::path::Path) -> Result> { .and_then(|bytes| serde_json::from_slice::<$ty>(&bytes).map_err(|e| e.to_string())) { Ok(value) => value, - Err(e) => { - warn!( - "Cache entry {} is unreadable ({}); ignoring and running fresh", - $path.display(), - e - ); - return Ok(None); + Err(reason) => { + return Miss(CacheMiss::Unreadable { + path: $path.clone(), + reason, + }) } } }; @@ -470,11 +643,7 @@ fn read_entry(dir: &std::path::Path) -> Result> { // A snapshot that cannot be replayed under the current semantics is a // miss, never an entry read as if it were current (#790). if let Err(reason) = snapshot.replayability() { - warn!( - "Cache entry {} is not reusable ({reason}); running fresh", - dir.display() - ); - return Ok(None); + return Miss(CacheMiss::NotReplayable(reason)); } let content_hash = content_hash(&snapshot); @@ -499,7 +668,7 @@ fn read_entry(dir: &std::path::Path) -> Result> { } }; - Ok(Some(AuditArtifacts { + Hit(Box::new(AuditArtifacts { fetch, snapshot, audit, @@ -950,13 +1119,74 @@ mod tests { with_big_stack(|| { let dir = tempfile::tempdir().unwrap(); write_entry(dir.path(), "AA", &entry(replayable_snapshot())).unwrap(); - let loaded = read_entry(dir.path()).unwrap().expect("compatible entry"); + let CacheLookup::Hit(loaded) = read_entry(dir.path()) else { + panic!("compatible entry expected"); + }; assert!(loaded.report.is_some()); assert!(!dir.path().join("report.json").exists()); assert!(replay(&loaded.snapshot).is_ok()); }); } + #[test] + fn a_snapshot_of_another_profile_is_not_the_asserted_measurement() { + with_big_stack(|| { + let mut snapshot = replayable_snapshot(); + let profile = &mut snapshot + .collected + .as_mut() + .unwrap() + .accessibility + .execution + .scope + .measurement_profile; + profile.state_id = Some("logged-in".to_string()); + profile.storage_state = Some(crate::audit::StorageStateRef { + sha256: "aa".repeat(32), + cookies: 1, + origins: 0, + }); + let ok = |e: ExpectedProfile| snapshot.check_measurement(&e); + // Nothing asserted, or the stored profile asserted: the same measurement. + assert!(ok(ExpectedProfile::default()).is_ok()); + assert!(ok(ExpectedProfile { + session_policy: Some("unknown".into()), + state_id: Some("logged-in".into()), + storage_state_sha256: Some("aa".repeat(32)), + }) + .is_ok()); + // Any asserted field that differs refuses, naming the field. + for (expected, field) in [ + ( + ExpectedProfile { + session_policy: Some("shared".into()), + ..Default::default() + }, + "session_policy", + ), + ( + ExpectedProfile { + state_id: Some("logged-out".into()), + ..Default::default() + }, + "state_id", + ), + ( + ExpectedProfile { + storage_state_sha256: Some("bb".repeat(32)), + ..Default::default() + }, + "storage_state", + ), + ] { + match ok(expected) { + Err(ReplayError::OtherMeasurement(d)) => assert_eq!(d[0].field, field), + other => panic!("{field}: {other:?}"), + } + } + }); + } + #[test] fn an_incompatible_or_legacy_entry_is_a_cache_miss() { with_big_stack(|| { @@ -964,7 +1194,10 @@ mod tests { let mut stale = replayable_snapshot(); stale.versions.as_mut().unwrap().evaluation += 1; write_entry(dir.path(), "AA", &entry(stale)).unwrap(); - assert!(read_entry(dir.path()).unwrap().is_none()); + assert!(matches!( + read_entry(dir.path()), + CacheLookup::Miss(CacheMiss::NotReplayable(ReplayError::Incompatible(_))) + )); // Written before the replay fields existed: the same. let legacy = tempfile::tempdir().unwrap(); @@ -974,7 +1207,11 @@ mod tests { &entry(snapshot_with_unlabelled_field(&[])), ) .unwrap(); - assert!(read_entry(legacy.path()).unwrap().is_none()); + let CacheLookup::Miss(reason) = read_entry(legacy.path()) else { + panic!("legacy entry must miss"); + }; + // The reason is named for the user, not swallowed. + assert!(reason.to_string().contains("incomplete"), "{reason}"); }); } } diff --git a/src/audit/mod.rs b/src/audit/mod.rs index bd8d51fa..15c06a38 100644 --- a/src/audit/mod.rs +++ b/src/audit/mod.rs @@ -36,8 +36,9 @@ pub mod validation; pub mod verdict; pub use artifacts::{ - cache_matches_signature, content_hash, hydrate_cached_report, load_artifacts, replay, - replay_report, save_artifacts, to_audit_report, AuditArtifacts, DocumentIdentity, + cache_matches_signature, cached_report, content_hash, hydrate_cached_report, load_artifacts, + lookup_artifacts, profile_differences, replay, replay_report, replay_report_as, save_artifacts, + to_audit_report, AuditArtifacts, CacheLookup, CacheMiss, DocumentIdentity, ExpectedProfile, FetchArtifact, ReplayError, SnapshotArtifact, SnapshotVersions, }; pub use batch::{ diff --git a/src/audit/support_matrix.rs b/src/audit/support_matrix.rs index 9027ddef..e3dbfece 100644 --- a/src/audit/support_matrix.rs +++ b/src/audit/support_matrix.rs @@ -31,7 +31,10 @@ pub enum Mode { Batch, /// `--reuse-cache` hit on a single URL: the stored report is returned. Cache, - /// Re-derivation from a stored snapshot (#790); there is no live page. + /// `auditmysite replay`: re-derivation from a stored snapshot (#790); there + /// is no live page. The report carries what the stored run measured — + /// including its extra checks — so a flag asking for a check *now* has + /// nothing to act on and is refused. Replay, } @@ -99,7 +102,10 @@ impl ExtraCheckSpec { } } -const REPLAY_DOES_NOT_MEASURE: Support = Support::NotRun("replay_does_not_measure"); +const REPLAY_DOES_NOT_MEASURE: Support = Support::Rejected( + "measures a live page; `replay` evaluates a stored snapshot and shows the checks its run \ + recorded (run the audit again to change them)", +); pub const MATRIX: &[ExtraCheckSpec] = &[ ExtraCheckSpec { @@ -159,6 +165,8 @@ pub struct ExtraCheckRequest { pub batch: bool, /// A single-URL run that may be answered from the cache. pub reuse_cache: bool, + /// The `replay` subcommand. + pub replay: bool, } impl ExtraCheckRequest { @@ -182,6 +190,12 @@ impl ExtraCheckRequest { /// Reject the first requested check this run cannot execute. pub fn validate(&self) -> Result<(), String> { for spec in MATRIX.iter().filter(|spec| self.requested(spec.check)) { + if self.replay { + if let Support::Rejected(reason) = spec.support(Mode::Replay) { + return Err(format!("{} {reason}", spec.flag)); + } + continue; + } if !self.prerequisite_met(spec.prerequisite) { if let Some(reason) = spec.prerequisite.unmet_reason() { return Err(format!("{} {reason}", spec.flag)); @@ -241,9 +255,26 @@ mod tests { #[test] fn a_replay_never_claims_to_have_measured() { + // Every check is refused in replay: it neither runs nor is recorded + // as not run over what the stored run measured. assert!(MATRIX .iter() - .all(|s| matches!(s.replay, Support::NotRun(_)))); + .all(|s| matches!(s.replay, Support::Rejected(_)))); + for flag in 0..4 { + let e = request(|r| { + r.replay = true; + match flag { + 0 => r.isolate_third_party_impact = true, + 1 => r.check_ssr_content = true, + 2 => r.ai_transparency = true, + _ => r.dns_check = true, + } + }) + .validate() + .unwrap_err(); + assert!(e.contains("replay"), "{e}"); + } + assert!(request(|r| r.replay = true).validate().is_ok()); } #[test] diff --git a/src/cli/args.rs b/src/cli/args.rs index 9e4cecfe..ff3d9095 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -489,6 +489,19 @@ pub enum Command { /// URL to plan an audit for (required unless --sitemap or --url-file is set) url: Option, }, + /// Evaluate a stored snapshot again — no browser, no network access. + /// + /// The target is a `snapshot.json`, a cache entry directory, or the URL + /// whose cached entry to replay. The report states `replay` as its + /// provenance. A snapshot that is incomplete or was written under other + /// versions is refused with its reason, never evaluated under today's + /// semantics. `--session`, `--state-id` and `--storage-state` assert the + /// measurement profile the snapshot must carry; another profile is + /// refused, not shown as the same measurement. + Replay { + /// Snapshot file, cache entry directory, or URL of a cached entry + target: String, + }, /// Compare the own `accname` computation against Chrome's native /// accessibility tree — a differential test without a screen reader. AccnameDiff { @@ -841,6 +854,20 @@ impl Args { } pub fn validate(&self) -> Result<(), String> { + // A replay evaluates a stored snapshot: a flag asking for a live + // check is refused, and a declared state must be readable (#788). + if matches!(self.command, Some(Command::Replay { .. })) { + self.load_storage_state()?; + return crate::audit::support_matrix::ExtraCheckRequest { + isolate_third_party_impact: self.isolate_third_party_impact, + check_ssr_content: self.check_ssr_content, + ai_transparency: self.ai_transparency, + dns_check: self.dns_check, + replay: true, + ..Default::default() + } + .validate(); + } // Subcommands don't need URL validation if self.command.is_some() { return Ok(()); @@ -920,6 +947,7 @@ impl Args { seo_active: full || self.seo, batch: is_batch, reuse_cache: self.reuse_cache && !self.force_refresh, + replay: false, } .validate()?; if self.technician && !is_batch { @@ -1347,6 +1375,26 @@ mod tests { assert_eq!(args.exclude_path, vec!["/a/tag/**"]); } + #[test] + fn replay_takes_a_target_and_refuses_live_check_flags() { + let args = Args::try_parse_from(["auditmysite", "replay", "https://example.com"]).unwrap(); + assert!( + matches!(args.command, Some(Command::Replay { ref target }) if target == "https://example.com") + ); + assert!(args.validate().is_ok()); + + let args = Args::try_parse_from([ + "auditmysite", + "--dns-check", + "replay", + "https://example.com", + ]) + .unwrap(); + let e = args.validate().unwrap_err(); + assert!(e.contains("--dns-check") && e.contains("replay"), "{e}"); + assert!(Args::try_parse_from(["auditmysite", "replay"]).is_err()); + } + #[test] fn accname_diff_nimmt_mehrere_urls_oder_url_datei() { let args = Args::try_parse_from([ diff --git a/src/cli/commands.rs b/src/cli/commands.rs index f365648b..8f092d99 100644 --- a/src/cli/commands.rs +++ b/src/cli/commands.rs @@ -24,6 +24,8 @@ pub(crate) async fn handle_command(command: &Command, args: &Args) -> Result run_plan_command(args, url.as_deref()), + // Handled in `run` (it yields a verdict); never dispatched here. + Command::Replay { .. } => unreachable!("replay is dispatched before handle_command"), Command::ReportLint { input, fail_on, diff --git a/src/cli/runners.rs b/src/cli/runners.rs index ec8f2428..6d03d9b7 100644 --- a/src/cli/runners.rs +++ b/src/cli/runners.rs @@ -10,13 +10,13 @@ use colored::Colorize; use dialoguer::Select; use tracing::info; -use auditmysite::audit::normalize; use auditmysite::audit::{ - analyze_crawl_links, analyze_sitemap_diagnostics_with_progress, cache_matches_signature, - crawl_site, hydrate_cached_report, load_artifacts, parse_sitemap, read_url_file, - run_concurrent_batch, run_single_audit, to_audit_report, BatchConfig, CrawlResult, - PipelineConfig, Verdict, + analyze_crawl_links, analyze_sitemap_diagnostics_with_progress, cached_report, crawl_site, + parse_sitemap, read_url_file, replay_report, run_concurrent_batch, run_single_audit, + BatchConfig, CacheMiss, CrawlResult, ExpectedProfile, PipelineConfig, SnapshotArtifact, + Verdict, }; +use auditmysite::audit::{normalize, normalize_checked}; use auditmysite::browser::{BrowserManager, BrowserOptions}; use auditmysite::cli::url_filter::{select_urls, stratified_sample}; use auditmysite::cli::{Args, OutputFormat, RequestMode, SampleStrategy}; @@ -109,72 +109,30 @@ async fn run_single(args: &Args, config: &Option) -> R if args.reuse_cache && !args.force_refresh { let expected_signature = PipelineConfig::from(args).audit_signature(); - match load_artifacts(url)? { - Some(cached) if cache_matches_signature(&cached.meta, &expected_signature) => { + // The stored snapshot is replayed (#790), not reconstructed: an entry + // that is incomplete, from other versions or for another audit + // configuration is a miss that says why, never today's semantics + // applied to old data. + match cached_report(url, &expected_signature)? { + Ok(report) => { if !args.quiet { println!( "{} {}", "Cache hit:".green().bold(), - "using cached audit artifacts".dimmed() + "replaying the stored snapshot".dimmed() ); } - - // Prefer the full cached report so every module section renders - // faithfully; fall back to the lossy reconstruction only for - // legacy entries written before report.json existed (#404). - let mut report = cached - .report - .clone() - .unwrap_or_else(|| to_audit_report(&cached, &args.lang)); - // The cached audit is judged under this run's profile, which - // is stored with the report, so every output and the exit - // code read the same decision (#781). - report.run_context.decision_profile = config - .as_ref() - .map(|c| c.effective_verdict_config().profile()) - .unwrap_or_default(); - let verdict_result = normalize(&report).normalized.decision; - // screen_reader_audit is #[serde(skip)] and therefore absent from - // the persisted report — rebuild it from the cached AXTree so the - // cached report renders the same sections as a fresh run (#404). - hydrate_cached_report(&mut report, &cached.snapshot, &args.lang); - - match args.effective_format() { - OutputFormat::Json => { - // Build the JSON from the hydrated full report (not the - // normalized-only `cached.audit`) so `detail.modules` - // carries the same module data the PDF renders (#404) — - // a cache hit must not drop the module blob. - output_single_report(&report, args)?; - output_screen_reader_sidecar(&report, args)?; - } - OutputFormat::Table - | OutputFormat::Pdf - | OutputFormat::Ai - | OutputFormat::Summary - | OutputFormat::Sarif => { - output_single_report(&report, args)?; - } - } - print_verdict(&verdict_result, args.quiet); - return Ok(RunOutcome { - verdict: verdict_result.verdict, - visual_only_urls: visual_only_urls(std::iter::once(( - report.url.as_str(), - report.accessibility.execution.display_modes.as_ref(), - ))), - }); + return emit_replayed(report, args, config); } - Some(_) if !args.quiet => { + Err(CacheMiss::Absent) => {} + Err(reason) if !args.quiet => { println!( "{} {}", "Cache skipped:".yellow().bold(), - "cached artifacts were produced with a different audit configuration — \ - running a fresh audit" - .dimmed() + format!("{reason} — running a fresh audit").dimmed() ); } - _ => {} + Err(_) => {} } } @@ -267,6 +225,74 @@ async fn run_single(args: &Args, config: &Option) -> R }) } +/// Decides and emits a report that came from a stored snapshot (cache hit or +/// `replay`): judged under this run's decision profile, which is stored with +/// the report, so every output and the exit code read the same decision +/// (#781). +fn emit_replayed( + mut report: auditmysite::AuditReport, + args: &Args, + config: &Option, +) -> Result { + report.run_context.decision_profile = config + .as_ref() + .map(|c| c.effective_verdict_config().profile()) + .unwrap_or_default(); + let verdict_result = normalize_checked(&report)?.normalized.decision; + output_single_report(&report, args)?; + if args.effective_format() == OutputFormat::Json { + output_screen_reader_sidecar(&report, args)?; + } + print_verdict(&verdict_result, args.quiet); + Ok(RunOutcome { + verdict: verdict_result.verdict, + visual_only_urls: visual_only_urls(std::iter::once(( + report.url.as_str(), + report.accessibility.execution.display_modes.as_ref(), + ))), + }) +} + +/// `auditmysite replay `: the stored snapshot, evaluated again +/// without browser or network. `session_from_cli` says whether `--session` +/// was given: its default is not an assertion about the snapshot. +pub(crate) fn run_replay_mode( + args: &Args, + config: &Option, + target: &str, + session_from_cli: bool, +) -> Result { + let path = std::path::Path::new(target); + let snapshot = if path.exists() { + SnapshotArtifact::read(path)? + } else if url::Url::parse(target).is_ok() { + SnapshotArtifact::read_cached(target)? + } else { + return Err(AuditError::ConfigError(format!( + "replay target '{target}' is neither an existing snapshot/cache directory nor a URL" + ))); + }; + let expected = ExpectedProfile { + session_policy: session_from_cli.then(|| args.session.as_str().to_string()), + state_id: args.state_id.clone(), + storage_state_sha256: args + .load_storage_state() + .map_err(AuditError::ConfigError)? + .map(|state| state.digest), + }; + let refuse = |e: auditmysite::audit::ReplayError| AuditError::ConfigError(e.to_string()); + snapshot.check_measurement(&expected).map_err(refuse)?; + let report = replay_report(&snapshot).map_err(refuse)?; + if !args.quiet { + println!( + "{} {}", + "Replay:".cyan().bold(), + format!("{} (no browser, no network)", report.url).dimmed() + ); + } + emit_replayed(report, args, config).map(|o| o.verdict) +} + async fn maybe_offer_sitemap_scan( args: &Args, url: &str, diff --git a/src/main.rs b/src/main.rs index 633a29e4..88558a49 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,7 @@ mod sitemap_suggest; use commands::{detect_chrome_command, handle_command}; use plan::print_banner; -use runners::{run_batch_mode, run_display_all, run_single_mode}; +use runners::{run_batch_mode, run_display_all, run_replay_mode, run_single_mode}; use std::io::{self, IsTerminal}; @@ -42,6 +42,8 @@ async fn main() { matches.value_source("interactive") == Some(clap::parser::ValueSource::CommandLine); let request_mode_from_cli = matches.value_source("request_mode") == Some(clap::parser::ValueSource::CommandLine); + let session_from_cli = + matches.value_source("session") == Some(clap::parser::ValueSource::CommandLine); let mut args = Args::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); setup_logging(&args); @@ -54,7 +56,7 @@ async fn main() { let report_mode = args.report_mode; let run_result = tokio::select! { - result = run(args, &config, request_mode_from_cli) => result, + result = run(args, &config, request_mode_from_cli, session_from_cli) => result, signal = shutdown_signal() => Err(AuditError::Interrupted { signal }), }; let exit_code = match run_result { @@ -129,9 +131,15 @@ fn setup_logging(args: &Args) { /// Main application logic async fn run( mut args: Args, - _config: &Option, + config: &Option, request_mode_from_cli: bool, + session_from_cli: bool, ) -> Result { + // A replay is the one subcommand with a verdict of its own. + if let Some(auditmysite::cli::Command::Replay { target }) = &args.command { + args.validate().map_err(AuditError::ConfigError)?; + return run_replay_mode(&args, config, target, session_from_cli); + } // Handle subcommands first if let Some(ref command) = args.command { return handle_command(command, &args) @@ -203,11 +211,11 @@ async fn run( let is_batch = args.sitemap.is_some() || args.url_file.is_some() || args.crawl; if args.display == Some(auditmysite::cli::DisplaySelection::All) { - run_display_all(&args, _config, is_batch).await + run_display_all(&args, config, is_batch).await } else if is_batch { - run_batch_mode(&args, _config).await + run_batch_mode(&args, config).await } else { - run_single_mode(&args, _config).await + run_single_mode(&args, config).await } } diff --git a/tests/fixtures/architecture/unchecked_normalize.txt b/tests/fixtures/architecture/unchecked_normalize.txt index 222f2694..ae10cdce 100644 --- a/tests/fixtures/architecture/unchecked_normalize.txt +++ b/tests/fixtures/architecture/unchecked_normalize.txt @@ -3,7 +3,7 @@ # CLI dispatchers and the exporters) or builds presentation from data that was checked; # the baseline may only shrink (#790). src/cli/report_writers.rs 2 -src/cli/runners.rs 2 +src/cli/runners.rs 1 src/output/ai.rs 1 src/output/builder/batch.rs 1 src/output/json.rs 1 diff --git a/tests/replay_paths_test.rs b/tests/replay_paths_test.rs new file mode 100644 index 00000000..7dd4525c --- /dev/null +++ b/tests/replay_paths_test.rs @@ -0,0 +1,278 @@ +//! One measurement, three ways back to a report (#790, #787, #788): the fresh +//! run, the `--reuse-cache` hit and `auditmysite replay` must give the same +//! canonical model — only the named volatile metadata may differ — and a +//! measurement taken under another profile (session policy, declared state) +//! must never be presented as the same one. +//! +//! Local fixture only; the cache lives in a throw-away `$HOME`. +//! +//! cargo test --test replay_paths_test -- --ignored + +mod common; + +use auditmysite::audit::normalized::NormalizedReport; +use auditmysite::audit::{ + cached_report, content_hash, normalize_checked, profile_differences, replay, run_single_audit, + CacheMiss, ExpectedProfile, ReplayError, SnapshotArtifact, +}; +use auditmysite::browser::StorageState; +use auditmysite::{BrowserManager, BrowserOptions, PipelineConfig, WcagLevel}; +use serde_json::Value; + +const PAGE: &str = "Replay fixture\ +

Replay fixture

Plain content.

"; + +/// A single-page app: the content changes after a navigation without a +/// reload (`pushState`), driven by the declared state in localStorage — the +/// state a login or a consent choice would leave. +const SPA: &str = "App\ +

Start

Welcome.

"; + +/// The cache directory is `$HOME/.auditmysite/cache`; keep the real one out of +/// the test. +fn isolated_home() { + static HOME: std::sync::OnceLock = std::sync::OnceLock::new(); + HOME.get_or_init(|| { + let dir = tempfile::tempdir().unwrap(); + std::env::set_var("HOME", dir.path()); + dir + }); +} + +fn config() -> PipelineConfig { + PipelineConfig { + wcag_level: WcagLevel::AA, + timeout_secs: 30, + stability_budget_ms: 1500, + verbose: false, + full_audit: false, + check_performance: false, + check_seo: true, + check_security: false, + check_mobile: false, + check_dark_mode: false, + check_design_quality: false, + check_html_conform: false, + check_ai_transparency: false, + check_dns: false, + check_isolate_third_party_impact: false, + check_ssr_content: false, + check_stack: false, + rule_filter: auditmysite::wcag::RuleFilterConfig::default(), + persist_artifacts: true, + capture_screenshots: false, + capture_element_evidence: false, + dismiss_consent: false, + exclude_selectors: Vec::new(), + interactive: auditmysite::cli::InteractiveMode::Off, + journey_budget_ms: auditmysite::a11y_journey::DEFAULT_BUDGET_MS, + session: Default::default(), + storage_state: None, + state_id: None, + lang: "en".to_string(), + display_mode: None, + indexability: Default::default(), + decision_profile: Default::default(), + } +} + +async fn browser() -> BrowserManager { + BrowserManager::with_options(BrowserOptions { + no_sandbox: std::env::var("CI").is_ok(), + ..Default::default() + }) + .await + .expect("browser") +} + +async fn fresh(url: &str, config: &PipelineConfig) -> auditmysite::AuditReport { + let browser = browser().await; + let report = run_single_audit(url, &browser, config).await; + browser.close().await.ok(); + report.expect("audit") +} + +/// The named volatile metadata is the only thing allowed to differ. +fn without_volatile(normalized: &NormalizedReport) -> Value { + let mut value = serde_json::to_value(normalized).unwrap(); + let object = value.as_object_mut().unwrap(); + object.remove("timestamp"); + object.remove("duration_ms"); + object["execution"]["environment"] + .as_object_mut() + .unwrap() + .remove("source"); + value +} + +/// A whole report is large and the audit future holds several: the default +/// test thread overflows its stack in an unoptimized build. +fn on_big_stack>(make: impl FnOnce() -> F + Send + 'static) { + std::thread::Builder::new() + .stack_size(128 << 20) + .spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(make()) + }) + .unwrap() + .join() + .unwrap(); +} + +#[test] +#[ignore = "needs Chrome"] +fn fresh_run_cache_hit_and_replay_give_one_canonical_model() { + on_big_stack(|| async { + isolated_home(); + let fixture = common::fixture_server::serve_html(PAGE.to_string()); + let config = config(); + let report = fresh(&fixture.url, &config).await; + assert!(fixture.served() > 0, "the fresh run measured the page"); + assert_eq!(report.accessibility.execution.environment.source, "live"); + + // From here on there is no page to ask. + fixture.stop(); + let served = fixture.served(); + + let fresh_model = normalize_checked(&report).unwrap().normalized; + let cached = cached_report(&fixture.url, &config.audit_signature()) + .unwrap() + .expect("cache hit"); + assert_eq!(cached.accessibility.execution.environment.source, "cache"); + let cache_model = normalize_checked(&cached).unwrap().normalized; + + let snapshot = SnapshotArtifact::read_cached(&fixture.url).unwrap(); + let replay_model = replay(&snapshot).unwrap(); + assert_eq!(replay_model.execution.environment.source, "replay"); + + let expected = without_volatile(&fresh_model); + assert_eq!(expected, without_volatile(&cache_model), "cache hit"); + assert_eq!(expected, without_volatile(&replay_model), "replay"); + assert!(fresh_model.findings.len() > 1, "a fixture with findings"); + assert_eq!(fixture.served(), served, "no network access after the run"); + }) +} + +#[test] +#[ignore = "needs Chrome"] +fn another_measurement_profile_is_not_the_same_measurement() { + on_big_stack(|| async { + isolated_home(); + let fixture = common::fixture_server::serve_html(SPA.to_string()); + let plain = config(); + let origin = fixture.url.clone(); + let declared = StorageState::from_bytes( + format!( + r#"{{"cookies":[],"origins":[{{"origin":"{origin}","localStorage":[{{"name":"route","value":"settings"}}]}}]}}"# + ) + .as_bytes(), + ) + .unwrap(); + let digest = declared.digest.clone(); + let in_state = PipelineConfig { + storage_state: Some(declared), + state_id: Some("settings-route".to_string()), + ..config() + }; + + // Same configuration twice: the same content, no difference to explain. + let first = fresh(&fixture.url, &plain).await; + let first_snapshot = SnapshotArtifact::read_cached(&fixture.url).unwrap(); + fresh(&fixture.url, &plain).await; + let again = SnapshotArtifact::read_cached(&fixture.url).unwrap(); + assert_eq!(content_hash(&first_snapshot), content_hash(&again)); + assert!(profile_differences(&profile(&first_snapshot), &profile(&again)).is_empty()); + + // The declared state changes what the page shows after its client-side + // navigation — and the profile names exactly that. + fresh(&fixture.url, &in_state).await; + let moved = SnapshotArtifact::read_cached(&fixture.url).unwrap(); + assert_ne!( + content_hash(&first_snapshot), + content_hash(&moved), + "the SPA changed its content without a reload" + ); + let explained = profile_differences(&profile(&first_snapshot), &profile(&moved)); + let fields: Vec<_> = explained.iter().map(|d| d.field).collect(); + assert!( + fields.contains(&"state_id") && fields.contains(&"storage_state"), + "{fields:?}" + ); + assert!( + first.accessibility.wcag_results.violations.len() != replay_violations(&moved), + "the state shows in the findings" + ); + + // The cache holds the in-state measurement now: the plain run is not + // answered with it, and the in-state run is. + assert!(matches!( + cached_report(&fixture.url, &plain.audit_signature()).unwrap(), + Err(CacheMiss::OtherConfiguration) + )); + assert!(cached_report(&fixture.url, &in_state.audit_signature()) + .unwrap() + .is_ok()); + + // A replay asserting another profile than the stored one is refused. + let asserts = |f: fn(&mut ExpectedProfile, &str), digest: &str| { + let mut expected = ExpectedProfile::default(); + f(&mut expected, digest); + moved.check_measurement(&expected) + }; + assert!(asserts(|_, _| {}, &digest).is_ok()); + assert!(asserts( + |e, d| { + e.state_id = Some("settings-route".into()); + e.storage_state_sha256 = Some(d.into()); + }, + &digest + ) + .is_ok()); + for refused in [ + asserts(|e, _| e.state_id = Some("logged-out".into()), &digest), + asserts(|e, _| e.session_policy = Some("shared".into()), &digest), + asserts( + |e, _| e.storage_state_sha256 = Some("0".repeat(64)), + &digest, + ), + ] { + assert!( + matches!(refused, Err(ReplayError::OtherMeasurement(_))), + "{refused:?}" + ); + } + fixture.stop(); + }) +} + +fn profile(snapshot: &SnapshotArtifact) -> auditmysite::audit::MeasurementProfile { + snapshot + .collected + .as_ref() + .unwrap() + .accessibility + .execution + .scope + .measurement_profile + .clone() +} + +fn replay_violations(snapshot: &SnapshotArtifact) -> usize { + snapshot + .collected + .as_ref() + .unwrap() + .accessibility + .wcag_results + .violations + .len() +}