From 14c8ded81c695c8357044e426591161d33d3ca31 Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:58:41 +0800 Subject: [PATCH] feat(chartstats): cache compiled measurement programs --- README.md | 8 +- .../ournotes-search/src/bin/ournotes-deck.rs | 27 +- crates/ournotes-search/tests/chartstats.rs | 3 + .../tests/chartstats/program_cache.rs | 170 ++++ crates/ournotes-sim/src/chartstats.rs | 69 +- crates/ournotes-sim/src/chartstats/cache.rs | 733 ++++++++++++++++++ .../src/chartstats/expectation.rs | 127 ++- crates/ournotes-sim/src/chartstats/luck.rs | 1 + .../src/live/full/chart_cache_identity.rs | 583 ++++++++++++++ crates/ournotes-sim/src/live/full/engine.rs | 10 + crates/ournotes-sim/src/live/full/luck_dp.rs | 45 ++ crates/ournotes-sim/src/live/full/mod.rs | 4 + .../ournotes-sim/src/live/full/scorecalc.rs | 28 + docs/chart-stats-cache.md | 88 +++ 14 files changed, 1848 insertions(+), 48 deletions(-) create mode 100644 crates/ournotes-search/tests/chartstats/program_cache.rs create mode 100644 crates/ournotes-sim/src/chartstats/cache.rs create mode 100644 crates/ournotes-sim/src/live/full/chart_cache_identity.rs create mode 100644 docs/chart-stats-cache.md diff --git a/README.md b/README.md index 669a562..4ee723f 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,8 @@ ournotes-deck live --data deck-data.json --roster box.json --score SCORE_ID --e 谱面统计： ```sh -ournotes-deck chart-stats --data deck-data.json [--seeds 8] [--charts ID,...] [--jobs N] -o chart-stats.json +ournotes-deck chart-stats --data deck-data.json [--seeds 8] [--charts ID,...] [--jobs N] \ + [--stats-cache DIR] -o chart-stats.json ``` 谱面统计格式为 `ournotes-deck.chart-stats/3`。激走开启时给出各次抽签、技能判定按名义概率独立的期望， @@ -193,6 +194,11 @@ ournotes-deck chart-stats --data deck-data.json [--seeds 8] [--charts ID,...] [- `--seeds N` 指定幸运谱面的回放种子数量，默认 8，与期望统计无关；没有幸运区间时回放种子为 `[0]`。 `--no-gekisou-aptitude` 关闭单技能适性，保留基线统计。 +`--stats-cache DIR` 持久保存各个已编译测量程序及抽签曲线的原始结果，分别复用基线、普通技能位置、 +激走变体与交叉项。每项完成后原子保存，可在中断后继续；输入的元数据、kind / shape 目录和校验卡组每次重新生成。 +缓存按模型源码和格式隔离，输出与无缓存测量一致；命中与计算计数写到 stderr。 +存储格式、依赖边界与 Rust API 见 [谱面统计缓存](docs/chart-stats-cache.md)。 + 每张谱面的 `expectation` 包含无技能 `score`、`scorePerfect`、区间结果、普通加分技能的 `weights[kind][position]`、 `rangeWeights[kind][position][range]`，以及随机卡组的 `check` / 固定名次的 `rankCheck`。 区间分数、名次奖励、幸运点数和四种抽签结果次数为期望区间；最大连击和 Just 数在证明与抽签独立后用整局模拟精确取得。 diff --git a/crates/ournotes-search/src/bin/ournotes-deck.rs b/crates/ournotes-search/src/bin/ournotes-deck.rs index ed19656..9f430b7 100644 --- a/crates/ournotes-search/src/bin/ournotes-deck.rs +++ b/crates/ournotes-search/src/bin/ournotes-deck.rs @@ -25,7 +25,7 @@ const USAGE: &str = "usage: ournotes-deck live --data FILE --roster FILE --score ID --expectation finite --seed-law FILE [--play FILE] [--gekisou] [common options] ournotes-deck chart-stats --data FILE [--seeds N] [--no-gekisou-aptitude] - [--charts ID[,ID...]] [--jobs N] [-o FILE] + [--charts ID[,ID...]] [--jobs N] [--stats-cache DIR] [-o FILE] recommend consumes the JSON recommendation request and writes the unified recommendation result. Use ournotes-deck recommend --help for its input and progress options. --data is a deck data file (nnnotes.deck-data/1). live ranks by the expected score of the whole-live simulation @@ -44,6 +44,7 @@ Estimates are [center, outward interval half-width] under independent nominal lo --seeds N controls replay seeds for charts with a luck range (default 8); statistics use nominal expectations. --no-gekisou-aptitude leaves both aptitude fields null and includes the baseline measurements. --charts keeps the listed score ids in file order; --jobs N measures N charts at once (default 1). +--stats-cache DIR reuses completed measurement programs and lottery propagations; counters go to stderr. -o FILE writes JSON to a file; without it JSON goes to stdout. common options: -k N (default 10), --leader ID, --include ID[,ID...], --exclude ID[,ID...], --exclude-snaps ID[,ID...], --no-snaps, --time-limit-ms N"; @@ -57,7 +58,7 @@ fn read(path: &str) -> Result { } fn chart_stats(args: &[String]) -> Result, String> { - let (mut data, mut seeds, mut out) = (None, None, None); + let (mut data, mut seeds, mut out, mut cache) = (None, None, None, None); let mut options = ournotes_sim::chartstats::Options::default(); let (mut only, mut jobs): (Option>, usize) = (None, 1); let mut i = 0; @@ -73,6 +74,7 @@ fn chart_stats(args: &[String]) -> Result, String> { "--no-gekisou-aptitude" => options.aptitude = false, "--charts" => only = Some(ids(&val()?)?), "--jobs" => jobs = val()?.trim().parse::().map_err(|_| "bad --jobs".to_string())?.max(1), + "--stats-cache" => cache = Some(val()?), "-o" | "--out" => out = Some(val()?), "-h" | "--help" => return Err(USAGE.into()), other => return Err(format!("unknown option {other}\n{USAGE}")), @@ -86,11 +88,20 @@ fn chart_stats(args: &[String]) -> Result, String> { if let Some(n) = seeds { options.replay_seeds = n; } + let cache = cache.map(ournotes_sim::chartstats::ChartStatsCache::new).transpose().map_err(|e| e.to_string())?; let doc = if jobs <= 1 { - ournotes_sim::chartstats::document_with(&data, &options).map_err(|e| e.to_string())? + match cache.as_ref() { + Some(cache) => ournotes_sim::chartstats::document_with_cache(&data, &options, cache), + None => ournotes_sim::chartstats::document_with(&data, &options), + } + .map_err(|e| e.to_string()) } else { - chart_stats_parallel(data, &options, jobs)? + chart_stats_parallel(data, &options, jobs, cache.as_ref()) }; + if let Some(cache) = cache { + eprintln!("chart cache {}", serde_json::to_string(&cache.snapshot()).expect("cache counters")); + } + let doc = doc?; match out { Some(path) => { let mut text = serde_json::to_string(&doc).expect("json"); @@ -107,6 +118,7 @@ fn chart_stats_parallel( mut data: DeckData, options: &ournotes_sim::chartstats::Options, jobs: usize, + cache: Option<&ournotes_sim::chartstats::ChartStatsCache>, ) -> Result { use ournotes_sim::chartstats; let charts = std::mem::take(&mut data.charts); @@ -122,8 +134,11 @@ fn chart_stats_parallel( let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let Some(c) = charts.get(i) else { break }; let t = std::time::Instant::now(); - let r = chartstats::chart_stats_with(&data.master, c, &kinds, options) - .map_err(|e| format!("chart {}: {e}", c.score_id)); + let r = match cache { + Some(cache) => chartstats::chart_stats_with_cache(&data.master, c, &kinds, options, cache), + None => chartstats::chart_stats_with(&data.master, c, &kinds, options), + } + .map_err(|e| format!("chart {}: {e}", c.score_id)); eprintln!( "chart {} {:.1} s{}", c.score_id, diff --git a/crates/ournotes-search/tests/chartstats.rs b/crates/ournotes-search/tests/chartstats.rs index 847daae..0cc0103 100644 --- a/crates/ournotes-search/tests/chartstats.rs +++ b/crates/ournotes-search/tests/chartstats.rs @@ -3,6 +3,9 @@ #[path = "../../ournotes-sim/tests/common/mod.rs"] mod common; +#[path = "chartstats/program_cache.rs"] +mod program_cache; + use ournotes_sim::chartstats::{self, POWER}; use ournotes_sim::data::{DeckData, FORMAT}; use ournotes_sim::live::full::{self, GekisouSetup, LiveNote, LiveParams, Performer}; diff --git a/crates/ournotes-search/tests/chartstats/program_cache.rs b/crates/ournotes-search/tests/chartstats/program_cache.rs new file mode 100644 index 0000000..5590d5a --- /dev/null +++ b/crates/ournotes-search/tests/chartstats/program_cache.rs @@ -0,0 +1,170 @@ +//! Native cache regressions: current documents agree byte for byte with independent measurements while +//! unchanged compiled programs remain reusable when the public kind and shape catalogs grow. + +use super::*; + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use chartstats::{ChartStatsCache, ChartStatsCacheStats, Options}; + +static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +struct Directory(PathBuf); + +impl Directory { + fn new() -> Self { + let time = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + Self(std::env::temp_dir().join(format!( + "ournotes-chart-cache-integration-{}-{time}-{}", + std::process::id(), + NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed) + ))) + } +} + +impl Drop for Directory { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn cached(data: &DeckData, path: &Path) -> (Value, ChartStatsCacheStats) { + // A new instance for every build proves that reuse comes from committed records on disk. + let cache = ChartStatsCache::new(path).unwrap(); + let doc = chartstats::document_with_cache(data, &Options::default(), &cache).unwrap(); + (doc, cache.snapshot()) +} + +fn assert_document_bytes(actual: &Value, expected: &Value) { + assert_eq!(serde_json::to_vec(actual).unwrap(), serde_json::to_vec(expected).unwrap()); +} + +fn incremental(before: &DeckData, after: &DeckData) -> (Value, Value, ChartStatsCacheStats) { + let directory = Directory::new(); + let (old, _) = cached(before, &directory.0); + let (current, reused) = cached(after, &directory.0); + let independent = chartstats::document_with(after, &Options::default()).unwrap(); + assert_document_bytes(¤t, &independent); + let cold_directory = Directory::new(); + let (cold_doc, cold) = cached(after, &cold_directory.0); + assert_document_bytes(¤t, &cold_doc); + assert!(reused.hits > 0, "unchanged programs must be reused: {reused:?}"); + assert!(reused.computed > 0, "changed programs must be measured: {reused:?}"); + assert!(reused.computed < cold.computed, "incremental {reused:?}, cold {cold:?}"); + assert_eq!(reused.invalid, 0); + (old, current, reused) +} + +#[test] +fn metadata_and_growth_changes_reuse_all_programs_with_current_headers() { + let before = aptitude_data(&APT_FEVERS); + let directory = Directory::new(); + let (old, first) = cached(&before, &directory.0); + assert!(first.computed > 0); + let (warm, second) = cached(&before, &directory.0); + assert_document_bytes(&warm, &old); + assert_eq!(second.computed, 0); + assert_eq!(second.writes, 0); + + let mut after = before.clone(); + after.provenance["master"]["version"] = json!("v2"); + after.provenance["exporter"]["version"] = json!("1"); + after.sha256 = Some("1".repeat(64)); + for row in &mut after.master.member_card_levels { + row.performance_rate += 17; + row.exp += 1; + } + for row in &mut after.master.support_card_levels { + row.visual_rate += 19; + } + let (current, reused) = cached(&after, &directory.0); + assert_eq!(reused.computed, 0, "{reused:?}"); + assert_eq!(reused.writes, 0); + assert_eq!(reused.invalid, 0); + assert!(reused.hits > 0); + assert_eq!(current["source"]["master"]["version"], "v2"); + assert_eq!(current["source"]["exporter"]["version"], "1"); + assert_eq!(current["charts"], old["charts"]); + assert_document_bytes(¤t, &chartstats::document_with(&after, &Options::default()).unwrap()); +} + +#[test] +fn adding_an_ordinary_kind_preserves_previous_programs() { + let before = aptitude_data(&APT_FEVERS); + let mut after = before.clone(); + let mut row = after.master.live_skill_effects[0].clone(); + row.id = after.master.live_skill_effects.iter().map(|r| r.id).max().unwrap() + 100; + row.live_skill_id = 90_001; + row.activation_time_second += 0.125; + after.master.live_skill_effects.push(row); + after.master.reindex().unwrap(); + assert_eq!(chartstats::kinds(&after.master).len(), chartstats::kinds(&before.master).len() + 1); + + let (old, current, _) = incremental(&before, &after); + let old_weights = old["charts"][0]["expectation"]["weights"].as_array().unwrap(); + let current_weights = current["charts"][0]["expectation"]["weights"].as_array().unwrap(); + assert_eq!(¤t_weights[..old_weights.len()], old_weights); + assert_eq!(current["gekisouAptitude"], old["gekisouAptitude"]); +} + +#[test] +fn changing_values_reuses_unit_weights_and_regenerates_checks() { + let before = aptitude_data(&APT_FEVERS); + let mut after = before.clone(); + for row in &mut after.master.live_skill_effects { + row.effect_value += 137; + } + let (old, current, _) = incremental(&before, &after); + assert_eq!(chartstats::kinds(&before.master).len(), chartstats::kinds(&after.master).len()); + assert_ne!(current["kinds"], old["kinds"]); + assert_eq!(current["charts"][0]["expectation"]["weights"], old["charts"][0]["expectation"]["weights"]); + assert_ne!(current["charts"][0]["expectation"]["check"], old["charts"][0]["expectation"]["check"]); +} + +#[test] +fn adding_a_member_shape_reuses_support_measurements_after_their_indices_move() { + let before = aptitude_data(&APT_FEVERS); + let old_shapes = chartstats::shapes(&before.master); + let old_support = shape_for(&old_shapes, "support", 1); + let mut after = before.clone(); + let mut skill = after.master.gekisou_skills.iter().find(|r| r.id == 1).unwrap().clone(); + skill.id = 90_001; + after.master.gekisou_skills.push(skill); + let mut effect = + after.master.gekisou_skill_effects.iter().find(|r| r.skill_id == 1 && r.level == 3).unwrap().clone(); + effect.id = after.master.gekisou_skill_effects.iter().map(|r| r.id).max().unwrap() + 100; + effect.skill_id = 90_001; + effect.effect_value += 2; + after.master.gekisou_skill_effects.push(effect); + // Another card still exposes the original skill of this duplicate card. + after.master.member_cards.last_mut().unwrap().gekisou_skill_id = 90_001; + after.master.reindex().unwrap(); + let current_shapes = chartstats::shapes(&after.master); + let current_support = shape_for(¤t_shapes, "support", 1); + assert_eq!(current_shapes.len(), old_shapes.len() + 1); + assert_eq!(current_support, old_support + 1); + + let (old, current, _) = incremental(&before, &after); + let variants = |doc: &Value, id: usize| { + doc["charts"][0]["gekisouAptitude"]["variants"] + .as_array() + .unwrap() + .iter() + .filter(|v| v["shape"] == id) + .cloned() + .collect::>() + }; + let old_variants = variants(&old, old_support); + let current_variants = variants(¤t, current_support); + assert_eq!(old_variants.len(), 2); + assert_eq!(current_variants.len(), 2); + assert!(old_variants.iter().zip(¤t_variants).any(|(a, b)| a["check"] != b["check"])); + for (mut a, mut b) in old_variants.into_iter().zip(current_variants) { + for key in ["shape", "check"] { + a.as_object_mut().unwrap().remove(key); + b.as_object_mut().unwrap().remove(key); + } + assert_document_bytes(&a, &b); + } +} diff --git a/crates/ournotes-sim/src/chartstats.rs b/crates/ournotes-sim/src/chartstats.rs index 2bbe0bf..fbbfbf5 100644 --- a/crates/ournotes-sim/src/chartstats.rs +++ b/crates/ournotes-sim/src/chartstats.rs @@ -16,6 +16,7 @@ use serde::Serialize; mod aptitude; +mod cache; mod expectation; mod luck; @@ -23,6 +24,7 @@ pub use aptitude::{ AptitudeHeader, ChartAptitude, Condition, Cumulative, Effect, RangeDelta, RangeFactors, Shape, ShapeSkill, Variant, aptitude_header, shapes, }; +pub use cache::{ChartStatsCache, ChartStatsCacheStats}; pub use expectation::{Estimate, ExpectationCheck, ExpectedRange, ExpectedStats}; pub use luck::{ LUCK_RUNS, LuckEntry, LuckOptions, LuckSteps, LuckTable, luck_compose, luck_neutral, luck_table_dp, @@ -387,6 +389,7 @@ struct Live<'m> { gekisou: Option, play: LivePlay, positions: usize, + cache: Option<&'m ChartStatsCache>, } /// A check deck: `(kind, value)` per position, and its effect rows `(live skill id, kind, value)`. @@ -512,8 +515,15 @@ impl Live<'_> { return Err(Error::Input("ranks without Gekisou".into())); } let mut lm = full::LiveModel::new(master, &deck, self.notes, self.events, params)?; - let score = lm.run(&play)?; - return Ok((score, Vec::new(), lm.converted_judgements())); + let identity = self.cache.and_then(|_| full::initialized_chart_program_identity(&mut lm, &play, None)); + let mut compute = || { + let score = lm.run(&play)?; + Ok((score, Vec::new(), lm.converted_judgements())) + }; + return match self.cache { + Some(cache) => cache.counted(identity.as_deref(), 0, compute), + None => compute(), + }; }; let mut lm = match ranks { None => full::LiveModel::new_gekisou(master, &deck, self.notes, self.events, params, &g.setup)?, @@ -526,8 +536,15 @@ impl Live<'_> { lm } }; - let score = lm.run_timed(&play, &g.dt)?; - Ok((score, lm.gekisou_ranges(), lm.converted_judgements())) + let identity = self.cache.and_then(|_| full::initialized_chart_program_identity(&mut lm, &play, Some(&g.dt))); + let mut compute = || { + let score = lm.run_timed(&play, &g.dt)?; + Ok((score, lm.gekisou_ranges(), lm.converted_judgements())) + }; + match self.cache { + Some(cache) => cache.counted(identity.as_deref(), g.setup.fevers.len().min(MAX_GEKISOU_FEVERS), compute), + None => compute(), + } } /// A master with these skills added, `(live skill id, kind, value)`, one effect row each. @@ -647,6 +664,28 @@ pub fn chart_stats_with( chart: &DataChart, kinds: &[Kind], options: &Options, +) -> Result { + chart_stats_impl(master, chart, kinds, options, None) +} + +/// [`chart_stats_with`] with persistent reuse of individual simulation programs. Current metadata and checks +/// are assembled on every call; this does not cache a chart document or a skill's global index. +pub fn chart_stats_with_cache( + master: &Master, + chart: &DataChart, + kinds: &[Kind], + options: &Options, + cache: &ChartStatsCache, +) -> Result { + chart_stats_impl(master, chart, kinds, options, Some(cache)) +} + +fn chart_stats_impl( + master: &Master, + chart: &DataChart, + kinds: &[Kind], + options: &Options, + cache: Option<&ChartStatsCache>, ) -> Result { options.validate()?; let settings = LiveScoreSettings::from_master(master)?; @@ -746,6 +785,7 @@ pub fn chart_stats_with( gekisou: None, play: JudgementStream::theoretical_best(&c).to_live_play()?, positions, + cache, }; let mut off_rng = Rng(OFF_CHECK_SALT ^ chart.score_id as u64); out.off_seeds.push(off.off_seed_stats(kinds, OFF_SEED, &mut off_rng, judged)?); @@ -772,6 +812,7 @@ pub fn chart_stats_with( gekisou: Some(Gekisou { setup, dt: stream.delta_times()?, perfect: perfect.to_live_play()? }), play: stream.to_live_play()?, positions, + cache, }; let setup = &live.gekisou.as_ref().expect("Gekisou on").setup; let luck = setup.missions.iter().take(setup.fevers.len()).any(|&m| m == MISSION_LUCK); @@ -815,11 +856,29 @@ pub fn document(data: &DeckData, replay_seeds: Option) -> Result Result { + document_impl(data, options, None) +} + +/// [`document_with`] with a persistent program cache. Header provenance, kinds and shapes always describe +/// `data`, including a run whose simulation programs all hit the cache. +pub fn document_with_cache( + data: &DeckData, + options: &Options, + cache: &ChartStatsCache, +) -> Result { + document_impl(data, options, Some(cache)) +} + +fn document_impl( + data: &DeckData, + options: &Options, + cache: Option<&ChartStatsCache>, +) -> Result { options.validate()?; let kinds = kinds(&data.master); let mut charts = Vec::with_capacity(data.charts.len()); for c in &data.charts { - charts.push(chart_stats_with(&data.master, c, &kinds, options).map_err(|e| match e { + charts.push(chart_stats_impl(&data.master, c, &kinds, options, cache).map_err(|e| match e { Error::Domain(m) => Error::Domain(format!("chart {}: {m}", c.score_id)), e => e, })?); diff --git a/crates/ournotes-sim/src/chartstats/cache.rs b/crates/ournotes-sim/src/chartstats/cache.rs new file mode 100644 index 0000000..0401477 --- /dev/null +++ b/crates/ournotes-sim/src/chartstats/cache.rs @@ -0,0 +1,733 @@ +//! Persistent results of individual chart measurement programs. Keys describe compiled programs, not chart +//! documents or master versions. Floating-point interval endpoints are stored by their original bit patterns. + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::value::RawValue; +use sha2::{Digest, Sha256}; + +use crate::Error; +use crate::live::certified::{F64Interval, ProbabilityMass}; +use crate::live::full::{ + GekisouRange, IntegerBounds, LuckDpCertifiedResult, LuckRangeMoments, LuckRangeScoreBounds, LuckScoreExpectation, + RealBounds, +}; + +const SCHEMA: &str = "ournotes-deck.chart-program-cache/1"; +const MAX_RECORD_BYTES: u64 = 64 * 1024 * 1024; +static TEMPORARY_ID: AtomicU64 = AtomicU64::new(0); + +/// Work performed through a chart program cache. `computed` counts successful computations, `writes` completed +/// atomic replacements, and `bytes` the bytes written. Failed computations are never stored. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ChartStatsCacheStats { + pub requests: u64, + pub hits: u64, + pub computed: u64, + pub writes: u64, + pub invalid: u64, + pub bytes: u64, +} + +#[derive(Default)] +struct Counters { + requests: AtomicU64, + hits: AtomicU64, + computed: AtomicU64, + writes: AtomicU64, + invalid: AtomicU64, + bytes: AtomicU64, +} + +/// A thread-safe, content-addressed cache of chart measurement programs. Each successful expectation, +/// deterministic run or lottery propagation is committed independently, so later failures preserve that work. +/// Input validation, current headers, kind/shape assembly and check-deck generation remain outside the cache. +pub struct ChartStatsCache { + directory: PathBuf, + counters: Counters, + locks: [Mutex<()>; 192], +} + +impl ChartStatsCache { + /// Open a cache directory. This schema has no migration from whole-chart statistics caches. Model source + /// identities occupy separate namespaces; identical compiled programs can be shared between master versions. + pub fn new(path: impl AsRef) -> Result { + let directory = path.as_ref().join("chart-program-v1").join(crate::SOURCE_SHA256); + fs::create_dir_all(&directory).map_err(|e| cache_error(&directory, e))?; + Ok(Self { directory, counters: Counters::default(), locks: std::array::from_fn(|_| Mutex::new(())) }) + } + + pub fn snapshot(&self) -> ChartStatsCacheStats { + let c = &self.counters; + ChartStatsCacheStats { + requests: c.requests.load(Ordering::Relaxed), + hits: c.hits.load(Ordering::Relaxed), + computed: c.computed.load(Ordering::Relaxed), + writes: c.writes.load(Ordering::Relaxed), + invalid: c.invalid.load(Ordering::Relaxed), + bytes: c.bytes.load(Ordering::Relaxed), + } + } + + pub(super) fn expectation( + &self, + identity: Option<&[u8]>, + ranges: usize, + compute: impl FnOnce() -> Result, + ) -> Result { + self.evaluate("expectation", identity, |v| v.ranges.len() == ranges, compute) + } + + pub(super) fn counted( + &self, + identity: Option<&[u8]>, + ranges: usize, + compute: impl FnOnce() -> Result, + ) -> Result { + self.evaluate("run", identity, |v| v.1.len() == ranges, compute) + } + + pub(crate) fn curve( + &self, + identity: Option<&[u8]>, + frames: usize, + probes: usize, + ranges: usize, + compute: impl FnOnce() -> Result, + ) -> Result { + self.evaluate( + "curve", + identity, + |v| v.probe_transitions.len() == frames && v.probes.len() == probes && v.range_moments.len() == ranges, + compute, + ) + } + + fn evaluate( + &self, + kind: &str, + identity: Option<&[u8]>, + valid: impl Fn(&T) -> bool, + compute: impl FnOnce() -> Result, + ) -> Result { + self.counters.requests.fetch_add(1, Ordering::Relaxed); + let Some(identity) = identity else { + let result = compute()?; + self.counters.computed.fetch_add(1, Ordering::Relaxed); + return Ok(result); + }; + let mut hash = Sha256::new(); + for part in + [SCHEMA.as_bytes(), crate::SOURCE_SHA256.as_bytes(), super::FORMAT.as_bytes(), kind.as_bytes(), identity] + { + hash.update((part.len() as u64).to_le_bytes()); + hash.update(part); + } + let hash = hash.finalize(); + let key = hex(&hash); + let directory = self.directory.join(kind).join(&key[..2]); + let path = directory.join(format!("{key}.json")); + // Different threads asking for the same program share its first result. Independent processes may + // duplicate computation, but use distinct temporary files and publish only complete records. + // An expectation may request a curve while its own lock is held; separate banks keep those nested + // operations independent even when their digest prefixes happen to agree. + let bank = match kind { + "expectation" => 0, + "run" => 1, + "curve" => 2, + _ => unreachable!(), + }; + let _lock = self.locks[bank * 64 + usize::from(hash[0]) % 64].lock().unwrap_or_else(|e| e.into_inner()); + if let Some(found) = self.read::(&path, kind, &key) { + if valid(&found) { + self.counters.hits.fetch_add(1, Ordering::Relaxed); + return Ok(found); + } + self.counters.invalid.fetch_add(1, Ordering::Relaxed); + } + let result = compute()?; + self.counters.computed.fetch_add(1, Ordering::Relaxed); + if !valid(&result) { + return Err(Error::Game("chart measurement returned an inconsistent cache record".into())); + } + let payload = serde_json::to_string(&result.stored()).map_err(|e| cache_error(&path, e))?; + let envelope = Envelope { + schema: SCHEMA.to_owned(), + source_sha256: crate::SOURCE_SHA256.to_owned(), + kind: kind.to_owned(), + key, + checksum: hex(&Sha256::digest(payload.as_bytes())), + payload: RawValue::from_string(payload).map_err(|e| cache_error(&path, e))?, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|e| cache_error(&path, e))?; + // Oversized programs remain valid measurements; bound the memory needed to read a cache record. + if bytes.len() as u64 <= MAX_RECORD_BYTES { + fs::create_dir_all(&directory).map_err(|e| cache_error(&directory, e))?; + write_atomic(&path, &bytes)?; + self.counters.writes.fetch_add(1, Ordering::Relaxed); + self.counters.bytes.fetch_add(bytes.len() as u64, Ordering::Relaxed); + } + Ok(result) + } + + fn read(&self, path: &Path, kind: &str, key: &str) -> Option { + let metadata = match fs::metadata(path) { + Ok(metadata) => metadata, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None, + Err(_) => { + self.counters.invalid.fetch_add(1, Ordering::Relaxed); + return None; + } + }; + let read = || { + if !metadata.is_file() || metadata.len() > MAX_RECORD_BYTES { + return None; + } + let bytes = fs::read(path).ok()?; + let envelope: Envelope = serde_json::from_slice(&bytes).ok()?; + if envelope.schema != SCHEMA + || envelope.source_sha256 != crate::SOURCE_SHA256 + || envelope.kind != kind + || envelope.key != key + || envelope.checksum != hex(&Sha256::digest(envelope.payload.get().as_bytes())) + { + return None; + } + T::restore(serde_json::from_str(envelope.payload.get()).ok()?) + }; + let result = read(); + if result.is_none() { + self.counters.invalid.fetch_add(1, Ordering::Relaxed); + } + result + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Envelope { + schema: String, + source_sha256: String, + kind: String, + key: String, + checksum: String, + payload: Box, +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(out, "{byte:02x}").expect("writing a string"); + } + out +} + +fn cache_error(path: &Path, error: impl std::fmt::Display) -> Error { + Error::Input(format!("chart statistics cache {}: {error}", path.display())) +} + +fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), Error> { + write_atomic_with(path, bytes, || TEMPORARY_ID.fetch_add(1, Ordering::Relaxed)) +} + +fn write_atomic_with(path: &Path, bytes: &[u8], mut next: impl FnMut() -> u64) -> Result<(), Error> { + let (temporary, mut file) = loop { + let temporary = path.with_extension(format!("{}.{}.tmp", std::process::id(), next())); + match OpenOptions::new().write(true).create_new(true).open(&temporary) { + Ok(file) => break (temporary, file), + // A process may have died before rename, and a later process may receive the same id. Its + // unfinished file is neither a cache hit nor owned by this writer; try another unique name. + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(cache_error(path, e)), + } + }; + let result = (|| { + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + fs::rename(&temporary, path) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result.map_err(|e| cache_error(path, e)) +} + +trait Record: Sized { + type Stored: Serialize + DeserializeOwned; + fn stored(&self) -> Self::Stored; + fn restore(stored: Self::Stored) -> Option; +} + +// DTOs contain integers only: neither JSON float parsing nor output rounding may change an interval endpoint. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Interval([u64; 2]); + +impl Interval { + fn of(value: RealBounds) -> Self { + Self([value.lower.to_bits(), value.upper.to_bits()]) + } + + fn bounds(self) -> Option { + let [lower, upper] = self.0.map(f64::from_bits); + F64Interval::new(lower, upper).ok().map(RealBounds::from) + } + + fn interval(self) -> Option { + let [lower, upper] = self.0.map(f64::from_bits); + F64Interval::new(lower, upper).ok() + } +} + +fn integers([lower, upper]: [i32; 2]) -> Option { + (lower <= upper).then_some(IntegerBounds { lower, upper }) +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExpectedRangeRecord { + range: usize, + start_query: Option, + end_query: usize, + percent: i64, + mean: Interval, + support: [i32; 2], + bonus_mean: Interval, + bonus_support: [i32; 2], + luck_points_mean: Option, + lot_results_mean: Option<[Interval; 4]>, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExpectationRecord { + final_mean: Interval, + final_support: [i32; 2], + ranges: Vec, +} + +impl Record for LuckScoreExpectation { + type Stored = ExpectationRecord; + + fn stored(&self) -> Self::Stored { + let Self { final_mean, final_support, ranges } = self; + ExpectationRecord { + final_mean: Interval::of(*final_mean), + final_support: [final_support.lower, final_support.upper], + ranges: ranges + .iter() + .map(|r| { + let LuckRangeScoreBounds { + range, + start_query, + end_query, + percent, + mean, + support, + bonus_mean, + bonus_support, + luck_points_mean, + lot_results_mean, + } = r; + ExpectedRangeRecord { + range: *range, + start_query: *start_query, + end_query: *end_query, + percent: *percent, + mean: Interval::of(*mean), + support: [support.lower, support.upper], + bonus_mean: Interval::of(*bonus_mean), + bonus_support: [bonus_support.lower, bonus_support.upper], + luck_points_mean: luck_points_mean.map(Interval::of), + lot_results_mean: lot_results_mean.map(|values| values.map(Interval::of)), + } + }) + .collect(), + } + } + + fn restore(stored: Self::Stored) -> Option { + let mut ranges = Vec::with_capacity(stored.ranges.len()); + for (index, r) in stored.ranges.into_iter().enumerate() { + if r.range != index { + return None; + } + ranges.push(LuckRangeScoreBounds { + range: r.range, + start_query: r.start_query, + end_query: r.end_query, + percent: r.percent, + mean: r.mean.bounds()?, + support: integers(r.support)?, + bonus_mean: r.bonus_mean.bounds()?, + bonus_support: integers(r.bonus_support)?, + luck_points_mean: Some(r.luck_points_mean?.bounds()?), + lot_results_mean: Some( + r.lot_results_mean? + .into_iter() + .map(Interval::bounds) + .collect::>>()? + .try_into() + .ok()?, + ), + }); + } + Some(Self { final_mean: stored.final_mean.bounds()?, final_support: integers(stored.final_support)?, ranges }) + } +} + +type Counted = (i32, Vec, u64); + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RangeRecord { + mission: i64, + state: u8, + combo: i32, + max_combo: i32, + just_count: i32, + start_score: i32, + end_score: i32, + luck_points: i32, + luck_gauge: i32, + rush_combo: i32, + lot_results: [i32; 4], + rank_bonus: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CountedRecord { + score: i32, + ranges: Vec, + converted: u64, +} + +impl Record for Counted { + type Stored = CountedRecord; + + fn stored(&self) -> Self::Stored { + CountedRecord { + score: self.0, + converted: self.2, + ranges: self + .1 + .iter() + .map(|r| { + let GekisouRange { + mission, + state, + combo, + max_combo, + just_count, + start_score, + end_score, + luck_points, + luck_gauge, + rush_combo, + lot_results, + rank_bonus, + } = *r; + RangeRecord { + mission, + state, + combo, + max_combo, + just_count, + start_score, + end_score, + luck_points, + luck_gauge, + rush_combo, + lot_results, + rank_bonus, + } + }) + .collect(), + } + } + + fn restore(stored: Self::Stored) -> Option { + let ranges = stored + .ranges + .into_iter() + .map(|r| { + (1..=8).contains(&r.state).then_some(GekisouRange { + mission: r.mission, + state: r.state, + combo: r.combo, + max_combo: r.max_combo, + just_count: r.just_count, + start_score: r.start_score, + end_score: r.end_score, + luck_points: r.luck_points, + luck_gauge: r.luck_gauge, + rush_combo: r.rush_combo, + lot_results: r.lot_results, + rank_bonus: r.rank_bonus, + }) + }) + .collect::>>()?; + Some((stored.score, ranges, stored.converted)) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CurveRecord { + probe_transitions: Vec, + steps: Vec<(i32, [Interval; 4])>, + probes: Vec, + range_moments: Vec<(Interval, [Interval; 4])>, + peak_states: usize, + transitions: u64, +} + +impl Record for LuckDpCertifiedResult { + type Stored = CurveRecord; + + fn stored(&self) -> Self::Stored { + let Self { probe_transitions, steps, probes, range_moments, peak_states, transitions } = self; + CurveRecord { + probe_transitions: probe_transitions.clone(), + steps: steps.iter().map(|(time, mass)| (*time, mass.map(|m| Interval::of(m.interval().into())))).collect(), + probes: probes.clone(), + range_moments: range_moments + .iter() + .map(|m| (Interval::of(m.luck_points.into()), m.lot_results.map(|v| Interval::of(v.into())))) + .collect(), + peak_states: *peak_states, + transitions: *transitions, + } + } + + fn restore(stored: Self::Stored) -> Option { + if stored.probe_transitions.iter().any(|&v| v > 15) { + return None; + } + let steps = stored + .steps + .into_iter() + .map(|(time, masses)| { + let masses = masses + .into_iter() + .map(|m| { + let bounds = m.bounds()?; + ProbabilityMass::from_bounds(bounds.lower, bounds.upper).ok() + }) + .collect::>>()? + .try_into() + .ok()?; + Some((time, masses)) + }) + .collect::>>()?; + let range_moments = stored + .range_moments + .into_iter() + .map(|(points, lots)| { + Some(LuckRangeMoments { + luck_points: points.interval()?, + lot_results: lots + .into_iter() + .map(Interval::interval) + .collect::>>()? + .try_into() + .ok()?, + }) + }) + .collect::>>()?; + Some(Self { + probe_transitions: stored.probe_transitions, + steps, + probes: stored.probes, + range_moments, + peak_states: stored.peak_states, + transitions: stored.transitions, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Directory(PathBuf); + + impl Directory { + fn new() -> Self { + let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + Self(std::env::temp_dir().join(format!( + "ournotes-chart-program-cache-{}-{stamp}-{}", + std::process::id(), + TEMPORARY_ID.fetch_add(1, Ordering::Relaxed) + ))) + } + } + + impl Drop for Directory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn record_path(cache: &ChartStatsCache, kind: &str) -> PathBuf { + let shard = fs::read_dir(cache.directory.join(kind)).unwrap().next().unwrap().unwrap().path(); + fs::read_dir(shard).unwrap().next().unwrap().unwrap().path() + } + + #[test] + fn expectation_and_probability_endpoints_round_trip_every_bit() { + let expected = LuckScoreExpectation { + final_mean: RealBounds { lower: -0.0, upper: f64::from_bits(1) }, + final_support: IntegerBounds { lower: 0, upper: 1 }, + ranges: vec![LuckRangeScoreBounds { + range: 0, + start_query: Some(2), + end_query: 5, + percent: 100, + mean: RealBounds { lower: 1.0, upper: 1.0f64.next_up() }, + support: IntegerBounds { lower: 1, upper: 2 }, + bonus_mean: RealBounds { lower: -1.0f64.next_up(), upper: 1.0 }, + bonus_support: IntegerBounds { lower: -2, upper: 1 }, + luck_points_mean: Some(RealBounds { lower: 0.0, upper: f64::from_bits(7) }), + lot_results_mean: Some([RealBounds { lower: 0.25f64.next_down(), upper: 0.25f64.next_up() }; 4]), + }], + }; + let bytes = serde_json::to_vec(&expected.stored()).unwrap(); + let restored = LuckScoreExpectation::restore(serde_json::from_slice(&bytes).unwrap()).unwrap(); + assert_eq!(serde_json::to_vec(&restored.stored()).unwrap(), bytes); + assert_eq!(restored.final_mean.lower.to_bits(), (-0.0f64).to_bits()); + + let curve = LuckDpCertifiedResult { + probe_transitions: vec![1, 3, 8], + steps: vec![(12, [ProbabilityMass::from_bounds(0.25f64.next_down(), 0.25f64.next_up()).unwrap(); 4])], + probes: vec![false, true], + range_moments: vec![LuckRangeMoments { + luck_points: F64Interval::new(1.0, 1.0f64.next_up()).unwrap(), + lot_results: [F64Interval::new(-0.0, f64::from_bits(1)).unwrap(); 4], + }], + peak_states: 17, + transitions: 100, + }; + let bytes = serde_json::to_vec(&curve.stored()).unwrap(); + let restored = LuckDpCertifiedResult::restore(serde_json::from_slice(&bytes).unwrap()).unwrap(); + assert_eq!(serde_json::to_vec(&restored.stored()).unwrap(), bytes); + } + + #[test] + fn a_new_instance_reuses_completed_programs_and_preserves_failures() { + let directory = Directory::new(); + let cache = ChartStatsCache::new(&directory.0).unwrap(); + cache.counted(Some(b"first"), 0, || Ok((12, Vec::new(), 3))).unwrap(); + let failure = Error::Domain("incomplete measurement".into()); + assert_eq!(cache.counted(Some(b"second"), 0, || Err(failure.clone())).unwrap_err(), failure); + assert_eq!(cache.snapshot().writes, 1); + let cache = ChartStatsCache::new(&directory.0).unwrap(); + assert_eq!(cache.counted(Some(b"first"), 0, || panic!("a completed run must be reused")).unwrap().0, 12); + cache.counted(Some(b"second"), 0, || Ok((34, Vec::new(), 4))).unwrap(); + assert_eq!(cache.snapshot().hits, 1); + assert_eq!(cache.snapshot().computed, 1); + assert_eq!(cache.snapshot().writes, 1); + } + + #[test] + fn damaged_checksum_or_payload_is_recomputed_and_repaired() { + let directory = Directory::new(); + let cache = ChartStatsCache::new(&directory.0).unwrap(); + cache.counted(Some(b"one"), 0, || Ok((12, Vec::new(), 3))).unwrap(); + let path = record_path(&cache, "run"); + let original = fs::read(&path).unwrap(); + for bad in [b"{truncated".to_vec(), b"[]".to_vec()] { + fs::write(&path, bad).unwrap(); + assert_eq!(cache.counted(Some(b"one"), 0, || Ok((12, Vec::new(), 3))).unwrap().0, 12); + assert_eq!(fs::read(&path).unwrap(), original); + } + let mut envelope: Envelope = serde_json::from_slice(&original).unwrap(); + envelope.payload = RawValue::from_string("{}".into()).unwrap(); + // A parseable payload is still invalid, both with a stale checksum and with a matching checksum. + for repair_checksum in [false, true] { + if repair_checksum { + envelope.checksum = hex(&Sha256::digest(envelope.payload.get().as_bytes())); + } + fs::write(&path, serde_json::to_vec(&envelope).unwrap()).unwrap(); + cache.counted(Some(b"one"), 0, || Ok((12, Vec::new(), 3))).unwrap(); + assert_eq!(fs::read(&path).unwrap(), original); + } + assert_eq!(cache.snapshot().invalid, 4); + assert_eq!(cache.snapshot().computed, 5); + } + + #[test] + fn concurrent_requests_commit_one_complete_record() { + let directory = Directory::new(); + let cache = ChartStatsCache::new(&directory.0).unwrap(); + let computed = AtomicU64::new(0); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + let result = cache + .counted(Some(b"shared"), 0, || { + computed.fetch_add(1, Ordering::Relaxed); + Ok((123, Vec::new(), 9)) + }) + .unwrap(); + assert_eq!(result.0, 123); + }); + } + }); + assert_eq!(computed.load(Ordering::Relaxed), 1); + assert_eq!(cache.snapshot().hits, 7); + assert_eq!(cache.snapshot().writes, 1); + let _: Envelope = serde_json::from_slice(&fs::read(record_path(&cache, "run")).unwrap()).unwrap(); + } + + #[test] + fn nested_curve_storage_does_not_hold_the_expectation_lock_bank() { + let directory = Directory::new(); + let cache = ChartStatsCache::new(&directory.0).unwrap(); + let empty_curve = || { + Ok(LuckDpCertifiedResult { + probe_transitions: Vec::new(), + steps: Vec::new(), + probes: Vec::new(), + range_moments: Vec::new(), + peak_states: 1, + transitions: 0, + }) + }; + let result = cache + .expectation(Some(b"expectation"), 0, || { + cache.curve(Some(b"curve"), 0, 0, 0, empty_curve)?; + Ok(LuckScoreExpectation { + final_mean: RealBounds { lower: 1.0, upper: 1.0 }, + final_support: IntegerBounds { lower: 1, upper: 1 }, + ranges: Vec::new(), + }) + }) + .unwrap(); + assert_eq!(result.final_mean.lower, 1.0); + assert_eq!(cache.snapshot().writes, 2); + } + + #[test] + fn an_interrupted_writer_with_a_reused_process_id_does_not_block_the_next_commit() { + let directory = Directory::new(); + fs::create_dir_all(&directory.0).unwrap(); + let path = directory.0.join("record.json"); + let stale = path.with_extension(format!("{}.7.tmp", std::process::id())); + fs::write(&stale, b"unfinished predecessor").unwrap(); + let mut ids = [7, 8].into_iter(); + write_atomic_with(&path, b"complete replacement", || ids.next().unwrap()).unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"complete replacement"); + assert_eq!(fs::read(&stale).unwrap(), b"unfinished predecessor"); + assert!(!path.with_extension(format!("{}.8.tmp", std::process::id())).exists()); + } +} diff --git a/crates/ournotes-sim/src/chartstats/expectation.rs b/crates/ournotes-sim/src/chartstats/expectation.rs index dc0b11f..1b13cba 100644 --- a/crates/ournotes-sim/src/chartstats/expectation.rs +++ b/crates/ournotes-sim/src/chartstats/expectation.rs @@ -206,49 +206,104 @@ impl<'a, 'm> Evaluator<'a, 'm> { let g = live.gekisou.as_ref().ok_or_else(|| Error::Input("expectation without Gekisou".into()))?; let play = if perfect { &g.perfect } else { &live.play }; let params = full::LiveParams { total_power: power, ..live.params }; - let key = format!("{formation:?}/{power}/{perfect}/{ranking:?}"); - let probability = match self.curves.get(&key) { - Some(value) => value.clone(), - None => { - let value = if g.setup.missions.iter().take(g.setup.fevers.len()).any(|&m| m == 2) { - full::luck_rush_dp_certified_with_moments( - self.master, - &self.skills, - live.notes, - live.events, - params, - &g.setup, - play, - &g.dt, - formation, - None, - ranking, - )? - } else { - LuckDpCertifiedResult { - probe_transitions: vec![1; play.frames.len()], - steps: Vec::new(), - probes: vec![false; self.skills.shapes.len()], - range_moments: vec![LuckRangeMoments::default(); g.setup.fevers.len()], - peak_states: 1, - transitions: 0, - } - }; - self.curves.entry(key).or_insert_with(|| Arc::new(value)).clone() - } - }; let deck: Vec<_> = (0..live.positions.max(formation.len()).max(1)) .map(|k| Performer { live_skill: live_skills.get(k).copied().flatten().map(|id| (id, 1)), ..formation.get(k).cloned().unwrap_or_default() }) .collect(); - let run = if full::has_nominal_score_probabilities(master, &deck, &self.skills) { - full::nominal_score_expectation_for_chart - } else { - full::luck_score_expectation_for_chart + let nominal = full::has_nominal_score_probabilities(master, &deck, &self.skills); + let identity = match live.cache { + Some(_) => full::chart_program_identity( + master, + &deck, + live.notes, + live.events, + params, + Some(&g.setup), + play, + Some(&g.dt), + ranking, + )? + .map(|mut identity| { + identity.push(u8::from(nominal)); + identity + }), + None => None, }; - run(master, &self.skills, &deck, live.notes, live.events, params, &g.setup, play, &g.dt, ranking, probability) + // Consult the complete program result before constructing its probability curve. A warm expectation + // therefore skips both probability propagation and score recording. + let mut compute = || { + let key = format!("{formation:?}/{power}/{perfect}/{ranking:?}"); + let probability = match self.curves.get(&key) { + Some(value) => value.clone(), + None => { + let value = if g.setup.missions.iter().take(g.setup.fevers.len()).any(|&m| m == 2) { + match live.cache { + Some(cache) => full::luck_rush_dp_certified_with_moments_cached( + self.master, + &self.skills, + live.notes, + live.events, + params, + &g.setup, + play, + &g.dt, + formation, + None, + ranking, + cache, + )?, + None => full::luck_rush_dp_certified_with_moments( + self.master, + &self.skills, + live.notes, + live.events, + params, + &g.setup, + play, + &g.dt, + formation, + None, + ranking, + )?, + } + } else { + LuckDpCertifiedResult { + probe_transitions: vec![1; play.frames.len()], + steps: Vec::new(), + probes: vec![false; self.skills.shapes.len()], + range_moments: vec![LuckRangeMoments::default(); g.setup.fevers.len()], + peak_states: 1, + transitions: 0, + } + }; + self.curves.entry(key).or_insert_with(|| Arc::new(value)).clone() + } + }; + let run = if nominal { + full::nominal_score_expectation_for_chart + } else { + full::luck_score_expectation_for_chart + }; + run( + master, + &self.skills, + &deck, + live.notes, + live.events, + params, + &g.setup, + play, + &g.dt, + ranking, + probability, + ) + }; + match live.cache { + Some(cache) => cache.expectation(identity.as_deref(), g.setup.fevers.len(), compute), + None => compute(), + } } #[allow(clippy::too_many_arguments)] diff --git a/crates/ournotes-sim/src/chartstats/luck.rs b/crates/ournotes-sim/src/chartstats/luck.rs index 3d0e6db..b75f7e5 100644 --- a/crates/ournotes-sim/src/chartstats/luck.rs +++ b/crates/ournotes-sim/src/chartstats/luck.rs @@ -486,6 +486,7 @@ pub fn diagnostic_luck_table( gekisou: Some(super::Gekisou { setup, dt: stream.delta_times()?, perfect: play.clone() }), play, positions: 5, + cache: None, }; live.luck_table(options) } diff --git a/crates/ournotes-sim/src/live/full/chart_cache_identity.rs b/crates/ournotes-sim/src/live/full/chart_cache_identity.rs new file mode 100644 index 0000000..abf19f5 --- /dev/null +++ b/crates/ournotes-sim/src/live/full/chart_cache_identity.rs @@ -0,0 +1,583 @@ +//! Source-versioned identities for the complete compiled programs used by chart statistics. +//! +//! A master/table digest is deliberately absent. The native constructors resolve targets, checker +//! topology, cumulative counters, phases, score constants and lottery tables before lookup. The +//! complete derived Debug representation keeps newly added model fields in the identity. Finite +//! Rust float Debug is round-tripping (including signed zero); nonfinite and opaque states decline +//! caching. The caller additionally namespaces these fingerprints by cache schema and simulator +//! source identity; this is not a stable serialization or a cross-engine equivalence claim. +//! +//! Raw row/skill identifiers have two observable roles: equality of state handles and execution +//! order. A private snapshot relabels handles bijectively, retaining aliases and the original order, +//! including the wrapped live-pool execution keys. The actual simulation is never relabeled. + +use std::collections::BTreeMap; +use std::fmt::{self, Write}; + +use sha2::{Digest, Sha256}; + +use super::{GekisouSetup, LiveModel, LiveNote, LiveParams, LivePlay, LuckScoreShape, LuckSource, Performer}; +use crate::{Error, master::Master, replay::RankConfirmation}; + +const IDENTITY_FORMAT: &str = "ournotes-deck.compiled-chart-program/1"; + +/// Hash Debug incrementally: even long charts' initialized frame storage does not allocate a huge +/// intermediate string or silently exceed a short-state cache limit. +struct IdentityWriter { + hash: Sha256, + tail: Vec, +} + +impl IdentityWriter { + fn new() -> Self { + Self { hash: Sha256::new(), tail: Vec::new() } + } + + fn finish(self) -> Vec { + self.hash.finalize().to_vec() + } +} + +impl Write for IdentityWriter { + fn write_str(&mut self, value: &str) -> fmt::Result { + // Tokens can straddle formatter writes. Refuse scripted checkers because their Rc alias + // topology is not represented by Debug; native chart constructors never create them. + const OPAQUE: [&[u8]; 4] = [b"NaN", b"inf", b"", b"Scripted"]; + let bytes = value.as_bytes(); + let mut boundary = self.tail.clone(); + boundary.extend_from_slice(&bytes[..bytes.len().min(9)]); + if OPAQUE.iter().any(|token| { + bytes.windows(token.len()).any(|part| part == *token) + || boundary.windows(token.len()).any(|part| part == *token) + }) { + return Err(fmt::Error); + } + self.hash.update(bytes); + if bytes.len() >= 9 { + self.tail.clear(); + self.tail.extend_from_slice(&bytes[bytes.len() - 9..]); + } else { + self.tail.extend_from_slice(bytes); + if self.tail.len() > 9 { + self.tail.drain(..self.tail.len() - 9); + } + } + Ok(()) + } +} + +fn label(labels: &mut BTreeMap, value: i64) -> i64 { + let next = labels.len() as i64; + *labels.entry(value).or_insert(next) +} + +fn write_schedule(hash: &mut Sha256, play: &LivePlay, delta_times: Option<&[f32]>) { + // Keep integer and float bits in a length-prefixed schedule. Exhaustive patterns make an added + // play/frame/judgement field a compile error rather than a silently omitted cache dependency. + let LivePlay { frames, base_seed } = play; + hash.update(base_seed.to_le_bytes()); + hash.update((frames.len() as u64).to_le_bytes()); + for frame in frames { + let super::PlayFrame { time_ms, judged } = frame; + hash.update(time_ms.to_le_bytes()); + hash.update((judged.len() as u64).to_le_bytes()); + for note in judged { + let super::JudgedNote { note_id, judgement, judgement_time_ms } = note; + hash.update(note_id.to_le_bytes()); + hash.update(judgement.to_le_bytes()); + hash.update(judgement_time_ms.to_le_bytes()); + } + } + match delta_times { + None => hash.update([0]), + Some(values) => { + hash.update([1]); + hash.update((values.len() as u64).to_le_bytes()); + for value in values { + hash.update(value.to_bits().to_le_bytes()); + } + } + } +} + +/// Opaque identity of a freshly constructed direct run. `None` means to run independently. The +/// seed is part of `play`; `None` delta times means `LiveModel::run`'s zero-delta frame schedule. +/// Queued ranks and all constructor-resolved state are already held by `model`. +pub(crate) fn initialized_chart_program_identity( + model: &mut LiveModel, + play: &LivePlay, + delta_times: Option<&[f32]>, +) -> Option> { + initialized_identity(model, play, delta_times, &()) +} + +fn initialized_identity( + model: &LiveModel, + play: &LivePlay, + delta_times: Option<&[f32]>, + context: &impl fmt::Debug, +) -> Option> { + if model.program_has_started + || model.raw_runtime.is_some() + || !model.enabled_live.is_empty() + || !model.guards.is_empty() + || !model.life_reductions.is_empty() + || !model.life_limits.is_empty() + || !model.gk_appliers.ids.is_empty() + || !model.gk_appliers.exhausted.is_empty() + || !model.gk_appliers.limit_finished.is_empty() + || !model.luck_suppressed.is_empty() + || model.score.calc.luck_weight.is_some() + || delta_times.is_some_and(|dt| dt.len() != play.frames.len() || dt.iter().any(|dt| !dt.is_finite())) + { + return None; + } + let mut snapshot = model.clone(); + let empty_score_frames = snapshot.score.compact_cache_snapshot()?; + + // Preserve the total preorder used by enabled_live.sort_by_key, including wrap-around and + // equal sort keys. Sorting just the raw skill ids would lose this observable native behavior. + let execution_keys: Vec<_> = + snapshot.live.iter().map(|skill| skill.key.wrapping_mul(10).wrapping_add(skill.index as i64)).collect(); + let mut distinct = execution_keys.clone(); + distinct.sort_unstable(); + distinct.dedup(); + let execution_order: Vec<_> = + execution_keys.iter().map(|key| distinct.binary_search(key).expect("key was collected")).collect(); + let mut row_labels = BTreeMap::new(); + for row in &mut snapshot.rows { + row.id = label(&mut row_labels, row.id); + } + let mut live_labels = BTreeMap::new(); + for skill in &mut snapshot.live { + skill.key = label(&mut live_labels, skill.key); + } + // Native pool order is retained, along with equality between each pool and its instances. + for pool in &mut snapshot.live_pools { + pool.key = label(&mut live_labels, pool.key); + } + let mut condition_labels = BTreeMap::new(); + for skill in &mut snapshot.cond { + skill.updater.canonicalize_cache_ids(&mut |id| label(&mut condition_labels, id)); + } + + // These are the only randomized lookup maps in a fresh judged-stream model. Sort their full + // contents for the key, keeping the caller's original storage and execution untouched. + let notes = std::mem::take(&mut snapshot.score.calc.note_factor_percent); + let judgements = std::mem::take(&mut snapshot.score.calc.judgement_score_factor_percent); + let ordered_notes: BTreeMap<_, _> = notes.into_iter().collect(); + let ordered_judgements: BTreeMap<_, _> = judgements.into_iter().collect(); + let mut out = IdentityWriter::new(); + write!( + out, + "{IDENTITY_FORMAT}/{:?}", + (snapshot, empty_score_frames, ordered_notes, ordered_judgements, execution_order, context) + ) + .ok()?; + write_schedule(&mut out.hash, play, delta_times); + Some(out.finish()) +} + +/// The extra master-dependent classification used by the nominal expectation/DP proof. The native +/// model intentionally omits ignored reset groups; `luck_skills` still classifies those raw groups. +/// Also retain DP's first raw-row mission lookup when a source contains repeated row identifiers. +/// Global shape numbers and probe representatives are replaced by the selected shape's contents. +type LotteryProfile<'a> = (Vec<(usize, LuckSource, Option<&'a LuckScoreShape>, Option)>, bool); + +/// Compile a chart's nominal expectation program before a cache lookup. This validates the complete +/// lottery catalog on every request, then retains only the held rows' resolved classification in the +/// key. Direct replay/off callers should use [`initialized_chart_program_identity`] on the model +/// they will execute, without invoking nominal-law admission for a seeded run. +#[allow(clippy::too_many_arguments)] +pub(crate) fn chart_program_identity( + master: &Master, + deck: &[Performer], + notes: &[LiveNote], + events: &[(i32, i32)], + params: LiveParams, + setup: Option<&GekisouSetup>, + play: &LivePlay, + delta_times: Option<&[f32]>, + ranking: Option<&[RankConfirmation]>, +) -> Result>, Error> { + if delta_times.is_some_and(|dt| dt.len() != play.frames.len()) { + return Err(Error::Input("one delta time per frame".into())); + } + let mut model = match (setup, ranking) { + (None, None) => LiveModel::new(master, deck, notes, events, params)?, + (None, Some(_)) => return Err(Error::Input("ranks without Gekisou".into())), + (Some(setup), None) => LiveModel::new_gekisou(master, deck, notes, events, params, setup)?, + (Some(setup), Some(ranking)) => { + let mut model = LiveModel::new_gekisou_ranked(master, deck, notes, events, params, setup)?; + model.set_rank_confirmation_timeline(ranking)?; + model + } + }; + let Some(setup) = setup else { + return Ok(initialized_chart_program_identity(&mut model, play, delta_times)); + }; + let skills = super::luck_skills(master)?; + let mut rows = Vec::new(); + for condition in &model.cond { + let source = match condition.skill_type { + super::SKILL_TYPE_GEKISOU => LuckSource::Gekisou, + super::SKILL_TYPE_GEKISOU_SUPPORT => LuckSource::GekisouSupport, + _ => continue, + }; + for effect in condition.updater.effects() { + let row = &model.rows[effect.row]; + let shape = skills.rows.get(&(source, row.id)).map(|&index| &skills.shapes[index].shape); + let mission = if shape.is_some() || super::is_luck_chain(row.effect_type) { + let native = source.rows(master).iter().find(|native| native.id == row.id); + native + .and_then(|native| match source { + LuckSource::Gekisou => master.gekisou_skill(native.skill_id), + LuckSource::GekisouSupport => master.gekisou_support_skill(native.skill_id), + }) + .map(|skill| skill.gekisou_mission_type) + } else { + None + }; + rows.push((effect.row, source, shape, mission)); + } + } + let profile: LotteryProfile<'_> = (rows, super::has_nominal_score_probabilities(master, deck, &skills)); + // Full setup also covers adapter admission beyond the native three-range storage. + Ok(initialized_identity(&model, play, delta_times, &(setup, profile))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::live::full::{JudgedNote, PlayFrame}; + use serde_json::json; + + struct Input { + master: Master, + deck: Vec, + notes: Vec, + events: Vec<(i32, i32)>, + params: LiveParams, + setup: GekisouSetup, + play: LivePlay, + dt: Vec, + } + + impl Input { + fn new() -> Self { + let lots: Vec<_> = + (0..5).map(|kind| json!({"_id":kind + 1,"_chanceLotType":kind,"_lotResult":3,"_weight":1})).collect(); + let tables = json!({ + "MasterLiveSettings": [ + {"_id":1,"_key":"note_score_adjustment_factor","_value":"3"}, + {"_id":2,"_key":"note_score_life_onus_factor","_value":"0.5"}, + {"_id":3,"_key":"life_base","_value":"1000"}, + {"_id":4,"_key":"life_denger","_value":"300"}, + {"_id":5,"_key":"gekisou_luck_gauge_max","_value":"140"}, + {"_id":6,"_key":"gekisou_luck_gauge_max_rush","_value":"70"}, + {"_id":7,"_key":"gekisou_luck_rush_score_bonus_percent","_value":"10"}], + "MasterLiveNoteParameter": [{"_id":1,"_noteOperateType":1,"_scorePercent":100}], + "MasterLiveJudgementParameter": [{"_id":1,"_noteSimulateJudgement":5,"_scorePercent":100,"_damage":0}], + "MasterLiveJudgementTiming": [{"_id":1,"_noteJudgementType":1,"_noteSimulateJudgement":5,"_afterMs":0}], + "MasterLiveGekisouLuckBasePoint": [{"_id":1,"_noteCategory":0,"_noteSimulateJudgement":5,"_weight":1,"_basePoint":60}], + "MasterLiveGekisouLuckBonusLot": lots, + "MasterSkillCondition": [ + {"_id":1,"_conditionType":5000,"_conditionValues":[],"_conditionTargetIDs":[1],"_isPositive":true}, + {"_id":2,"_conditionType":7021,"_conditionValues":[],"_conditionTargetIDs":[],"_isPositive":true}], + "MasterSkillConditionSet": [ + {"_id":1,"_group":1,"_conditionIds":[1]}, + {"_id":2,"_group":2,"_conditionIds":[2]}], + "MasterSkillTarget": [{"_id":1,"_skillTargetType":1,"_bandID":1}], + "MasterGekisouSkill": [{"_id":10,"_gekisouMissionType":2}], + "MasterGekisouSkillEffect": [{"_id":100,"_gekisouSkillID":10,"_level":1, + "_skillTriggerType":2,"_skillTriggerConditionGroup":2,"_skillConditionGroup":1, + "_skillReleaseConditionGroup":0,"_skillTargetIDs":[],"_skillEffectType":2000, + "_activationTimeSecond":0.0,"_effectValue":10000,"_maxEffectValue":0, + "_effectLimitCount":0,"_skillCumulativeConditionID":0,"_effectExecuteLimitCount":0, + "_effectExecuteLimitResetConditionGroup":0}], + "MasterLiveSkillEffect": [{"_id":1000,"_liveSkillID":-1000000,"_level":1, + "_skillConditionGroup":0,"_skillReleaseConditionGroup":0,"_skillTargetIDs":[], + "_skillEffectType":2000,"_activationTimeSecond":5.0,"_effectValue":10000, + "_maxEffectValue":0,"_effectLimitCount":0,"_skillCumulativeConditionID":0, + "_effectExecuteLimitCount":0,"_effectExecuteLimitResetConditionGroup":0}] + }); + let texts: Vec<_> = tables + .as_object() + .unwrap() + .iter() + .map(|(name, rows)| (name.clone(), json!({"_allData":rows}).to_string())) + .collect(); + let master = Master::from_json_tables(|name| { + texts.iter().find(|(key, _)| key == name).map(|(_, value)| value.as_str()) + }) + .unwrap(); + let notes = vec![LiveNote { note_id: 1, time_ms: 100, note_operate_type: 1, judgement_type: 1 }]; + Self { + master, + deck: vec![Performer { band_id: 1, gekisou_skill: Some((10, 1)), ..Default::default() }], + notes, + events: vec![(0, 0)], + params: LiveParams { + total_power: 300000, + music_level: 20, + converted_note_count: 1, + music_length_ms: 1000, + score_music_length_ms: None, + assist_factor: 1.0, + skill_target_music_type: 0, + }, + setup: GekisouSetup { fevers: vec![(0, 200)], missions: vec![2, 2, 2] }, + play: LivePlay { + base_seed: 0, + frames: vec![ + PlayFrame { time_ms: 0, judged: vec![] }, + PlayFrame { + time_ms: 100, + judged: vec![JudgedNote { note_id: 1, judgement: 5, judgement_time_ms: 100 }], + }, + PlayFrame { time_ms: 1200, judged: vec![] }, + ], + }, + dt: vec![0.1; 3], + } + } + + fn identity(&self) -> Vec { + chart_program_identity( + &self.master, + &self.deck, + &self.notes, + &self.events, + self.params, + Some(&self.setup), + &self.play, + Some(&self.dt), + None, + ) + .unwrap() + .unwrap() + } + + fn model(&self) -> LiveModel { + LiveModel::new_gekisou(&self.master, &self.deck, &self.notes, &self.events, self.params, &self.setup) + .unwrap() + } + } + + #[test] + fn unrelated_master_rows_and_new_shapes_do_not_invalidate_a_held_program() { + let mut input = Input::new(); + let expected = input.identity(); + let mut row = input.master.gekisou_skill_effects[0].clone(); + row.id = 1; + row.skill_id = 1; + row.activation_time_second = 1.0; + input.master.gekisou_skills.push(crate::master::SkillRow { + id: 1, + gekisou_mission_type: 2, + ..Default::default() + }); + input.master.gekisou_skill_effects.push(row); + let mut ordinary = input.master.live_skill_effects[0].clone(); + ordinary.id = 999999; + ordinary.live_skill_id = 99; + ordinary.effect_value = 70000; + input.master.live_skill_effects.push(ordinary); + input.master.reindex().unwrap(); + assert_eq!(input.identity(), expected); + assert_eq!(super::super::luck_skills(&input.master).unwrap().shapes.len(), 2); + } + + #[test] + fn representative_ids_and_synthetic_kind_indices_are_not_measurement_dependencies() { + let mut input = Input::new(); + input.deck[0].live_skill = Some((-1000000, 1)); + let expected = input.identity(); + input.master.gekisou_skills[0].id = 900; + input.master.gekisou_skill_effects[0].skill_id = 900; + input.master.gekisou_skill_effects[0].id = 9000; + input.deck[0].gekisou_skill = Some((900, 1)); + input.master.live_skill_effects[0].live_skill_id = -1000003; + input.master.live_skill_effects[0].id = 40000; + input.deck[0].live_skill = Some((-1000003, 1)); + input.master.reindex().unwrap(); + assert_eq!(input.identity(), expected); + } + + #[test] + fn resolved_targets_conditions_and_phase_changes_invalidate() { + let mut input = Input::new(); + let expected = input.identity(); + input.master.skill_targets[0].band_id = 2; + input.master.reindex().unwrap(); + assert_ne!(input.identity(), expected); + input.master.skill_targets[0].band_id = 1; + input.master.skill_conditions[0].is_positive = false; + input.master.reindex().unwrap(); + assert_ne!(input.identity(), expected); + input.master.skill_conditions[0].is_positive = true; + input + .master + .skill_effect_settings + .push(serde_json::from_value(json!({"_id":1,"_skillEffectType":2000,"_phase":2})).unwrap()); + input.master.reindex().unwrap(); + assert_ne!(input.identity(), expected); + } + + #[test] + fn ignored_native_reset_groups_still_change_the_lottery_proof_identity() { + let mut input = Input::new(); + input.master.gekisou_skill_effects[0].skill_trigger_condition_group = 0; + input.master.reindex().unwrap(); + let expected = input.identity(); + let mut before = input.model(); + let direct = initialized_chart_program_identity(&mut before, &input.play, Some(&input.dt)); + input.master.gekisou_skill_effects[0].effect_execute_limit_reset_condition_group = 2; + input.master.reindex().unwrap(); + let mut after = input.model(); + assert_eq!(initialized_chart_program_identity(&mut after, &input.play, Some(&input.dt)), direct); + assert_ne!(input.identity(), expected); + } + + #[test] + fn nominal_identity_keeps_the_dp_first_source_row_lookup() { + let mut input = Input::new(); + let expected = input.identity(); + let mut before = input.model(); + let direct = initialized_chart_program_identity(&mut before, &input.play, Some(&input.dt)); + // The held skill still compiles exactly the same. DP additionally finds the first raw row + // by id in the whole source and then reads that row's skill mission; do not skip that read. + input.master.gekisou_skills.push(crate::master::SkillRow { + id: 1, + gekisou_mission_type: 1, + ..Default::default() + }); + let mut other = input.master.gekisou_skill_effects[0].clone(); + other.skill_id = 1; + input.master.gekisou_skill_effects.insert(0, other); + input.master.reindex().unwrap(); + let mut after = input.model(); + assert_eq!(initialized_chart_program_identity(&mut after, &input.play, Some(&input.dt)), direct); + assert_ne!(input.identity(), expected); + } + + #[test] + fn native_effect_order_and_counterfactual_rank_inputs_are_retained() { + let mut input = Input::new(); + let mut second = input.master.gekisou_skill_effects[0].clone(); + second.id += 1; + second.effect_value += 5000; + input.master.gekisou_skill_effects.push(second); + input.master.reindex().unwrap(); + let expected = input.identity(); + input.master.gekisou_skill_effects[0].id += 2; + assert_ne!(input.identity(), expected, "raw id ordering determines native effect order"); + let ranked = |rank, percent| { + chart_program_identity( + &input.master, + &input.deck, + &input.notes, + &input.events, + input.params, + Some(&input.setup), + &input.play, + Some(&input.dt), + Some(&[RankConfirmation { frame: 0, range: 0, rank, percent }]), + ) + .unwrap() + .unwrap() + }; + assert_ne!(ranked(1, 10), ranked(2, 10)); + assert_ne!(ranked(1, 10), ranked(1, 20)); + assert_ne!(ranked(1, 10), input.identity()); + } + + #[test] + fn row_aliases_and_wrapped_execution_order_are_retained() { + let mut input = Input::new(); + input.deck[0].live_skill = Some((-1000000, 1)); + input.deck.push(Performer { live_skill: Some((11, 1)), ..Default::default() }); + let mut second = input.master.live_skill_effects[0].clone(); + second.live_skill_id = 11; + second.id = 1001; + input.master.live_skill_effects.push(second); + input.master.reindex().unwrap(); + let expected = input.identity(); + input.master.live_skill_effects[1].id = 1000; + assert_ne!(input.identity(), expected, "row-id equality is shared by conversion contexts"); + input.master.live_skill_effects[1].id = 1001; + input.master.live_skill_effects[1].live_skill_id = -1000001; + input.deck[1].live_skill = Some((-1000001, 1)); + assert_ne!(input.identity(), expected, "live-pool order is observable"); + + let mut a = input.model(); + let mut b = a.clone(); + // Keep pool membership/order identical but change the wrapped enabled-live order only. + for skill in b.live.iter_mut().filter(|skill| skill.member == 0) { + skill.key = i64::MAX / 10 + 1; + } + for pool in b.live_pools.iter_mut().filter(|pool| pool.member == 0) { + pool.key = i64::MAX / 10 + 1; + } + assert_ne!( + initialized_chart_program_identity(&mut a, &input.play, Some(&input.dt)), + initialized_chart_program_identity(&mut b, &input.play, Some(&input.dt)) + ); + } + + #[test] + fn finite_float_bits_and_full_schedule_are_preserved_nonfinite_states_decline() { + let input = Input::new(); + let mut model = input.model(); + let mut keys = std::collections::BTreeSet::new(); + for value in + [0.0f32, -0.0, f32::from_bits(1), f32::MIN_POSITIVE, 1.0, f32::from_bits(1.0f32.to_bits() + 1), f32::MAX] + { + model.score.calc.assist_factor = value; + assert!(keys.insert(initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)).unwrap())); + } + for value in [f32::INFINITY, f32::NEG_INFINITY, f32::NAN, f32::from_bits(0xffc00001)] { + model.score.calc.assist_factor = value; + assert!(initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)).is_none()); + } + model.score.calc.assist_factor = 1.0; + let expected = initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)); + let mut play = input.play.clone(); + play.base_seed = 1; + assert_ne!(initialized_chart_program_identity(&mut model, &play, Some(&input.dt)), expected); + play = input.play.clone(); + play.frames[1].judged[0].judgement = 6; + assert_ne!(initialized_chart_program_identity(&mut model, &play, Some(&input.dt)), expected); + let mut dt = input.dt.clone(); + dt[0] = f32::from_bits(dt[0].to_bits() + 1); + assert_ne!(initialized_chart_program_identity(&mut model, &input.play, Some(&dt)), expected); + dt[0] = f32::NAN; + assert!(initialized_chart_program_identity(&mut model, &input.play, Some(&dt)).is_none()); + } + + #[test] + fn random_lookup_iteration_is_sorted_and_execution_model_is_untouched() { + let input = Input::new(); + let mut model = input.model(); + model.score.calc.note_factor_percent.extend([(2, 90), (3, 80)]); + let before = format!("{model:?}"); + let expected = initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)); + assert_eq!(format!("{model:?}"), before); + let entries: Vec<_> = model.score.calc.note_factor_percent.drain().collect(); + model.score.calc.note_factor_percent.extend(entries.into_iter().rev()); + assert_eq!(initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)), expected); + model.score.calc.note_factor_percent.insert(2, 91); + assert_ne!(initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)), expected); + model.frame_timed(0, &[], 0.1).unwrap(); + assert!(initialized_chart_program_identity(&mut model, &input.play, Some(&input.dt)).is_none()); + } + + #[test] + fn opaque_tokens_split_between_formatter_writes_decline() { + for parts in [["N", "aN"], ["-i", "nf"], [""], ["Scr", "ipted"]] { + let mut out = IdentityWriter::new(); + out.write_str(parts[0]).unwrap(); + assert!(out.write_str(parts[1]).is_err()); + } + } +} diff --git a/crates/ournotes-sim/src/live/full/engine.rs b/crates/ournotes-sim/src/live/full/engine.rs index 4071e49..b9ad030 100644 --- a/crates/ournotes-sim/src/live/full/engine.rs +++ b/crates/ournotes-sim/src/live/full/engine.rs @@ -405,6 +405,16 @@ impl ConditionSkillUpdater { &self.effects } + /// Relabel identities on a private cache-key snapshot, never on an executing model. Keep the + /// already compiled order and all aliases: native wrapping may make that order differ from row + /// order, and distinct updater pools can share the same actual state key. + pub(super) fn canonicalize_cache_ids(&mut self, label: &mut impl FnMut(i64) -> i64) { + for effect in &mut self.effects { + effect.effect_id = label(effect.effect_id); + } + self.execute_count = self.execute_count.drain().map(|(id, count)| (label(id), count)).collect(); + } + /// Installs semantic probability checks on the selected score rows before recording. Replacing /// probability leaves preserves every trigger/reset property used to compile the idle plan. pub(super) fn condition_score_probabilities(&mut self, score_rows: &[bool]) -> Result { diff --git a/crates/ournotes-sim/src/live/full/luck_dp.rs b/crates/ournotes-sim/src/live/full/luck_dp.rs index f0ee908..17d6c99 100644 --- a/crates/ournotes-sim/src/live/full/luck_dp.rs +++ b/crates/ournotes-sim/src/live/full/luck_dp.rs @@ -1272,6 +1272,51 @@ pub fn luck_rush_dp_certified_with_moments( certified_mode(master, skills, notes, skill_events, params, setup, play, delta_times, deck, probes, ranking, true) } +/// Chart measurements persist the propagation by the complete recorded transcript. Recording still validates +/// the current dependency closure; a hit may share the same curve between different ordinary score programs. +#[allow(clippy::too_many_arguments)] +pub(crate) fn luck_rush_dp_certified_with_moments_cached( + master: &Master, + skills: &LuckSkills, + notes: &[LiveNote], + skill_events: &[(i32, i32)], + params: LiveParams, + setup: &GekisouSetup, + play: &LivePlay, + delta_times: &[f32], + deck: &[Performer], + probes: Option<&[Option]>, + ranking: Option<&[crate::replay::RankConfirmation]>, + cache: &crate::chartstats::ChartStatsCache, +) -> Result { + let transcript = record::( + master, + skills, + notes, + skill_events, + params, + setup, + play, + delta_times, + deck, + probes, + ranking, + true, + )?; + let identity = transcript.key().map(|words| words.into_iter().flat_map(u64::to_le_bytes).collect::>()); + cache.curve(identity.as_deref(), play.frames.len(), transcript.probes.len(), setup.fevers.len(), || { + let result = propagate(&transcript)?; + Ok(LuckDpCertifiedResult { + probe_transitions: result.probe_transitions, + steps: result.steps, + probes: result.probes, + range_moments: result.range_moments, + peak_states: result.peak_states, + transitions: result.transitions, + }) + }) +} + #[allow(clippy::too_many_arguments)] pub(super) fn certified_mode( master: &Master, diff --git a/crates/ournotes-sim/src/live/full/mod.rs b/crates/ournotes-sim/src/live/full/mod.rs index 1ea7657..9bc5c80 100644 --- a/crates/ournotes-sim/src/live/full/mod.rs +++ b/crates/ournotes-sim/src/live/full/mod.rs @@ -38,6 +38,8 @@ //! types are [`Error::Unsupported`] (4004 of Gekisou and Gekisou support skills keeps its no-op on a judged stream). mod applier_plan; +mod chart_cache_identity; +pub(crate) use chart_cache_identity::{chart_program_identity, initialized_chart_program_identity}; mod combo; #[cfg(feature = "search-diagnostics")] pub use applier_plan::with_applier_plan_disabled; @@ -56,6 +58,7 @@ pub use gekisou::{ mod life; mod luck; mod luck_dp; +pub(crate) use luck_dp::luck_rush_dp_certified_with_moments_cached; mod luck_exact; pub use luck_dp::{ LuckDpCache, LuckDpCacheStats, LuckDpCertifiedResult, LuckDpResult, LuckRangeMoments, LuckRecordProfile, @@ -68,6 +71,7 @@ pub use luck_exact::{ LuckExactStats, luck_exact_law_with_ranking, }; mod luck_score_bounds; +pub(crate) use luck_score_bounds::IntegerBounds; pub(crate) use luck_score_bounds::luck_score_expectation_for_chart; mod nominal_expectation; pub use luck_score_bounds::{ diff --git a/crates/ournotes-sim/src/live/full/scorecalc.rs b/crates/ournotes-sim/src/live/full/scorecalc.rs index 84989bd..31f027f 100644 --- a/crates/ournotes-sim/src/live/full/scorecalc.rs +++ b/crates/ournotes-sim/src/live/full/scorecalc.rs @@ -36,6 +36,15 @@ fn mill(m: i32) -> f32 { } impl FrameDiff { + fn is_cache_zero(&self) -> bool { + // Exhaustive destructuring forces new fields to be reviewed before empty frame storage can + // be compacted in a cache-key snapshot. In particular, negative zero is not positive zero. + let Self { band_total_power, combo, note, just, perfect, great, good, luck } = *self; + band_total_power == 0 + && luck == 0 + && [combo, note, just, perfect, great, good].into_iter().all(|value| value.to_bits() == 0) + } + fn add(&mut self, cmd: &FactorCommand) { self.band_total_power = self.band_total_power.wrapping_add(cmd.band_total_power); self.luck = self.luck.wrapping_add(cmd.luck); @@ -137,6 +146,25 @@ pub(crate) struct IncrementalCalculator { } impl IncrementalCalculator { + /// Compact only untouched frame storage in a private identity snapshot. Original lengths are + /// part of that identity; all other calculator fields still use their complete representation. + /// Never call this on a model that will execute. + pub(super) fn compact_cache_snapshot(&mut self) -> Option<[usize; 3]> { + if self.program.is_some() + || self.bounds_trace.is_some() + || self.notes.iter().any(|frame| !frame.is_empty()) + || self.factors.iter().any(|frame| !frame.is_empty()) + || self.diffs.iter().any(|diff| !diff.is_cache_zero()) + { + return None; + } + let lengths = [self.notes.len(), self.factors.len(), self.diffs.len()]; + self.notes.clear(); + self.factors.clear(); + self.diffs.clear(); + Some(lengths) + } + pub(crate) fn new(calc: LiveScoreCalculator, music_length_ms: i32) -> IncrementalCalculator { let max_frame = get_frame(music_length_ms) + EXTRA_FRAMES; let n = max_frame as usize; diff --git a/docs/chart-stats-cache.md b/docs/chart-stats-cache.md new file mode 100644 index 0000000..5c72243 --- /dev/null +++ b/docs/chart-stats-cache.md @@ -0,0 +1,88 @@ +# Chart statistics program cache + +`chart-stats` can retain the expensive results of individual measurement programs between builds: + +```sh +ournotes-deck chart-stats --data deck-data.json --jobs 4 \ + --stats-cache build/chart-program-cache -o chart-stats.json +``` + +The output remains `ournotes-deck.chart-stats/3`. It contains nominal expectations under independent lottery and +skill probabilities. `--seeds` controls the separate replay seed list; it does not control the precision or work +budget of these expectations. + +## What is reused + +The cache stores three kinds of completed computation: + +| Record | Reusable work | +| --- | --- | +| Expectation | One compiled formation and ordinary-skill program at a specified power, play and rank schedule | +| Run | A deterministic whole-live run, including its integer score, range counters and converted judgements | +| Curve | Lottery-state propagation for the complete transcript recorded by the reduced native interpreter | + +No-skill baselines, each ordinary kind at each position, each Gekisou skill variant, and their plain-kind cross +terms therefore reuse work independently. Adding a skill does not by itself invalidate the other programs. +Changing an effect or one of its resolved condition, target, cumulative, timing or score dependencies changes +the program identity. The probability tier can share a curve between different score programs when their +recorded lottery transcripts are identical. + +An expectation cache hit is checked before probability construction. A warm build skips both propagation and +score recording for that program. A curve hit still records the current interpreter transcript before reusing +its propagation result. + +Every build reads and validates its current input and creates the current metadata, kinds and shapes. Shape +indices, skill membership lists and check decks are assembled again. If an index or a real skill value changes +the generated check deck, that check uses its current program identity. The cache does not preserve an old +chart header or copy a check from a differently indexed skill. + +## Identities and storage + +Program identities describe the initialized interpreter and its play schedule, including resolved formation +predicates and condition checkers. Source identifiers that only name interpreter objects are normalized while +their alias relationships and execution order are retained. Expectation identities also retain the relevant +lottery classification used by the nominal expectation evaluator. Curve identities use the complete recorded +transcript, including the current probe layout. + +Every key includes the cache schema, chart statistics format and model source SHA-256. A new model source uses +a separate namespace. The cache does not import older whole-chart statistics records. It never searches other +master versions for an approximately matching result. + +The payloads store original integer results and binary64 interval endpoints as integer bit patterns. Decimal +rounding happens only when assembling the final statistics document, exactly as in a run without a cache. +Each record carries its key, model identity, schema and payload SHA-256. Truncated, structurally invalid or +checksum-mismatched records are misses and are replaced after a successful computation. A record larger than +64 MiB is computed normally without being persisted. + +Completed records are written to unique temporary files, flushed and atomically renamed before the next +computation returns. An error or interruption preserves all earlier committed programs. One cache instance +can be shared by chart workers; requests for the same key are synchronized. Separate processes may compute +the same missing program concurrently, but publish complete records through independent temporary files. +The cache is disposable: deleting its directory changes the amount of work, not the output. +Records are retained across input changes and there is no automatic garbage collection. This preserves programs +that another chart, region or later input can still reuse. Operators can remove an obsolete source namespace or +the complete cache directory when reclaiming storage. + +## Rust API + +The original functions remain available. Call the corresponding cache-aware entry point to retain programs: + +```rust +use ournotes_sim::chartstats::{self, ChartStatsCache, Options}; + +# fn build(data: &ournotes_sim::data::DeckData) -> Result<(), ournotes_sim::Error> { +let cache = ChartStatsCache::new("build/chart-program-cache")?; +let document = chartstats::document_with_cache(data, &Options::default(), &cache)?; +let counters = cache.snapshot(); +# let _ = (document, counters); +# Ok(()) +# } +``` + +For parallel callers, `chart_stats_with_cache(master, chart, kinds, options, &cache)` measures one chart using +the same shared instance. `chart_stats_with` and `document_with` remain the uncached entry points. + +`snapshot()` reports `requests`, `hits`, `computed`, `writes`, `invalid` and `bytes`. Requests count calls through +the three cache tiers; computed counts successful computations, writes counts committed records, and bytes +counts their serialized size. Unsupported or failed computations are not stored and are not successful +computations. The CLI prints these counters to stderr, leaving the JSON output unchanged.