diff --git a/README.md b/README.md index 9ab8b251..5cec5e9c 100644 --- a/README.md +++ b/README.md @@ -340,10 +340,21 @@ provider-backed ELF evidence was required. non-pass rows; PageIndex/OpenKB use existing fixture/source provenance, while Honcho remains a source-provenance research gate until product-runtime evidence is checked in. - The full ELF fixture run scores 75 pass, 0 wrong_result, 0 unsupported claims, and + The full ELF fixture run scores 77 pass, 0 wrong_result, 0 unsupported claims, and 7 blocked jobs with aggregate evidence/source-ref/quote coverage at 1.000, while every public product row remains non-comparable until held-out, leakage-audited, digest-identified product-runtime evidence exists. +- R1 model ladder and local background organizer gate after XY-1163: the July 4 + follow-up adds `cargo make real-world-memory-r1-local-organizer`, defines the + L0/L1/L2/L3/L4 model ladder, and scores one fixture-backed proposal-only local + organizer job. The gate reports extraction F1 as explicitly `not_encoded` until a + labeled extraction set exists, validates JSON/schema, citation/source-ref coverage, + unsupported-claim rate, stale/correction/delete behavior, zero source mutation, + zero silent Memory Authority mutation, escalation rate, mean/p95 latency, and + tier-level cost/resource/runtime provenance. This is not evidence that small/local + models are safe defaults for Memory Authority writes or that ELF has Dreaming, + Letta, LangMem/LangChain, mem0/OpenMemory, Graphiti/Zep, or hosted managed-memory + parity. - Operator-approved public-proxy addendum after XY-930: the June 19 follow-up runs `cargo make baseline-production-private-addendum` with a simulated/public-proxy production corpus manifest approved for this stage. The run records 12 documents, @@ -493,6 +504,7 @@ Detailed evidence and interpretation: - [Source-Backed Memory Quality Benchmark Harness - July 3, 2026](docs/evidence/benchmarking/2026-07-03-source-backed-quality-benchmark-harness.md) - [qmd Candidate-Replay Comparability Gate - July 3, 2026](docs/evidence/benchmarking/2026-07-03-qmd-candidate-replay-comparability-gate.md) - [Final Source-Backed Project Memory Closeout Report - July 3, 2026](docs/evidence/benchmarking/2026-07-03-final-source-backed-project-memory-closeout-report.md) +- [R1 Local Background Organizer Benchmark Gate - July 4, 2026](docs/evidence/benchmarking/2026-07-04-r1-local-background-organizer-benchmark-gate.md) - [Live Baseline Benchmark Runbook](docs/runbook/benchmarking/live_baseline_benchmark.md) - [Single-User Production Runbook](docs/runbook/single_user_production.md) - Benchmark contract: @@ -603,7 +615,7 @@ Detailed comparison, mechanism-level analysis, and source map: - [Derived Knowledge Page Follow-Up Research](docs/research/derived_knowledge_page_followup.md) - [Dreaming Product Surface Follow-Up Research](docs/research/dreaming_product_surface_followup.md) -Latest real-world benchmark report: July 3, 2026. Latest external research refresh: +Latest real-world benchmark report: July 4, 2026. Latest external research refresh: June 11, 2026; June 20 adds the Agent Knowledge OS Closeout Benchmark Report, the Graph Topic-Map Report - June 20, 2026, Knowledge Workspace Version-Diff Report - June 20, 2026, and the Live Knowledge-Page Rebuild/Lint Report - June 20, @@ -622,6 +634,8 @@ June 27 adds the public quantitative competitor scoreboard report with row-level comparability gates and no universal leaderboard claim. July 3 adds the source-backed memory quality benchmark harness, the qmd candidate-replay comparability gate, and the final source-backed project memory closeout report. +July 4 adds the R1 model ladder and proposal-only local background organizer +benchmark gate. ## Documentation diff --git a/apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json b/apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json new file mode 100644 index 00000000..a9a79676 --- /dev/null +++ b/apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json @@ -0,0 +1,317 @@ +{ + "schema": "elf.real_world_job/v1", + "job_id": "local-background-organizer-proposal-only-001", + "suite": "local_background_organizer", + "title": "Local background organizer drafts proposals without authority mutation", + "corpus": { + "corpus_id": "real-world-memory-r1-local-background-organizer-2026-07-04", + "profile": "synthetic", + "items": [ + { + "evidence_id": "local-organizer-source-current", + "kind": "work_journal_entry", + "text": "Current session: the operator wants a local background organizer to draft low-risk memory proposals only, with source refs and no automatic Memory Authority write.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_memory_fixture/v1", + "ref": { + "fixture": "local_background_organizer_proposal_only_gate", + "evidence_id": "local-organizer-source-current" + } + }, + "created_at": "2026-07-04T10:00:00Z" + }, + { + "evidence_id": "local-organizer-conflict-policy", + "kind": "policy", + "text": "Policy: conflict resolution, multi-source synthesis, promotion review, important corrections, benchmark judging, and public claims require a strong model or explicit policy gate.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_memory_fixture/v1", + "ref": { + "fixture": "local_background_organizer_proposal_only_gate", + "evidence_id": "local-organizer-conflict-policy" + } + }, + "created_at": "2026-07-04T10:01:00Z" + }, + { + "evidence_id": "local-organizer-stale-delete", + "kind": "audit", + "text": "Audit: stale, correction, and delete observations are scored before promotion and must stay reviewable proposals until a reviewer or stronger gate accepts the change.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_memory_fixture/v1", + "ref": { + "fixture": "local_background_organizer_proposal_only_gate", + "evidence_id": "local-organizer-stale-delete" + } + }, + "created_at": "2026-07-04T10:02:00Z" + }, + { + "evidence_id": "local-organizer-decoy", + "kind": "decoy", + "text": "Decoy: the local background organizer already promoted every extracted preference directly into current Memory Authority.", + "source_ref": { + "schema": "source_ref/v1", + "resolver": "real_world_memory_fixture/v1", + "ref": { + "fixture": "local_background_organizer_proposal_only_gate", + "evidence_id": "local-organizer-decoy" + } + }, + "created_at": "2026-07-04T10:03:00Z" + } + ], + "adapter_response": { + "adapter_id": "fixture_local_background_organizer", + "answer": { + "content": "The local background organizer drafted one reviewable proposal with source refs and no Memory Authority write. It escalated conflict resolution, multi-source synthesis, promotion review, important corrections, benchmark judging, and public claims to a strong model or explicit policy gate. Stale, correction, and delete observations were scored before promotion and kept as reviewable proposals.", + "claims": [ + { + "claim_id": "proposal_only", + "text": "The local background organizer drafted one reviewable proposal with source refs and no Memory Authority write.", + "evidence_ids": [ + "local-organizer-source-current" + ], + "confidence": "high" + }, + { + "claim_id": "strong_gate_escalation", + "text": "Conflict resolution, multi-source synthesis, promotion review, important corrections, benchmark judging, and public claims were escalated to a strong model or explicit policy gate.", + "evidence_ids": [ + "local-organizer-conflict-policy" + ], + "confidence": "high" + }, + { + "claim_id": "stale_correction_delete_reviewable", + "text": "Stale, correction, and delete observations were scored before promotion and kept as reviewable proposals.", + "evidence_ids": [ + "local-organizer-stale-delete" + ], + "confidence": "high" + } + ], + "evidence_ids": [ + "local-organizer-source-current", + "local-organizer-conflict-policy", + "local-organizer-stale-delete" + ], + "latency_ms": 42.0, + "cost": { + "currency": "USD", + "amount": 0.0, + "input_tokens": 320, + "output_tokens": 180 + } + }, + "consolidation": { + "proposals": [ + { + "proposal_id": "proposal-local-organizer-review-only", + "proposal_kind": "local_background_organization_candidate", + "source_refs": [ + "local-organizer-source-current", + "local-organizer-conflict-policy", + "local-organizer-stale-delete" + ], + "expected_source_refs": [ + "local-organizer-source-current", + "local-organizer-conflict-policy", + "local-organizer-stale-delete" + ], + "usefulness_score": 0.91, + "min_usefulness_score": 0.8, + "expected_review_action": "defer", + "actual_review_action": "defer", + "source_mutations": [], + "unsupported_claim_count": 0, + "unsupported_claim_flags": [], + "diff": { + "summary": "Stage a local organizer proposal for human or strong-gate review without applying it.", + "before": {}, + "after": { + "target": "reviewable_local_background_organizer_proposal", + "apply_intent": "no_op", + "authority_write_allowed": false + } + } + } + ] + }, + "local_organizer": { + "model_tier": "L2", + "model_profile_id": "local_background_organizer_l2", + "runtime_id": "fixture_schema_constrained_local_runtime", + "runtime_commit": "fixture-runtime@2026-07-04", + "reproducibility_provenance": [ + "apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json", + "cargo make real-world-memory-r1-local-organizer" + ], + "extraction_f1": null, + "extraction_f1_blocker": "not_encoded: no labeled extraction set is checked in for this first R1 local organizer slice.", + "json_schema_valid": true, + "citation_source_ref_coverage": 1.0, + "unsupported_claim_rate": 0.0, + "stale_correction_delete_score": 1.0, + "source_mutation_count": 0, + "silent_memory_authority_mutation_count": 0, + "required_escalation_count": 2, + "completed_escalation_count": 2, + "resource_footprint": { + "runtime": "fixture", + "device": "local_cpu", + "peak_memory_mb": 64, + "network_required": false + } + } + } + }, + "timeline": [ + { + "event_id": "local-organizer-r1-scope", + "ts": "2026-07-04T10:00:00Z", + "actor": "operator", + "action": "set_scope", + "evidence_ids": [ + "local-organizer-source-current", + "local-organizer-conflict-policy", + "local-organizer-stale-delete" + ], + "summary": "The R1 slice was constrained to reviewable proposals with strong-gate escalation for authority-changing work." + } + ], + "prompt": { + "role": "user", + "content": "Run the local background organizer gate and report only reviewable proposals; do not mutate Memory Authority.", + "job_mode": "local_background_organizer", + "constraints": [ + "proposal_only", + "cite_evidence", + "escalate_authority_changes", + "report_schema_and_resource_metrics" + ] + }, + "expected_answer": { + "must_include": [ + { + "claim_id": "proposal_only", + "text": "The local background organizer drafted one reviewable proposal with source refs and no Memory Authority write." + }, + { + "claim_id": "strong_gate_escalation", + "text": "Conflict resolution, multi-source synthesis, promotion review, important corrections, benchmark judging, and public claims were escalated to a strong model or explicit policy gate." + }, + { + "claim_id": "stale_correction_delete_reviewable", + "text": "Stale, correction, and delete observations were scored before promotion and kept as reviewable proposals." + } + ], + "must_not_include": [ + "already promoted every extracted preference directly into current Memory Authority" + ], + "evidence_links": { + "proposal_only": "local-organizer-source-current", + "strong_gate_escalation": "local-organizer-conflict-policy", + "stale_correction_delete_reviewable": "local-organizer-stale-delete" + }, + "answer_type": "proposal_gate_report", + "accepted_alternates": [], + "requires_caveat": false, + "requires_refusal": false + }, + "required_evidence": [ + { + "evidence_id": "local-organizer-source-current", + "claim_id": "proposal_only", + "requirement": "cite", + "quote": "draft low-risk memory proposals only" + }, + { + "evidence_id": "local-organizer-conflict-policy", + "claim_id": "strong_gate_escalation", + "requirement": "cite", + "quote": "require a strong model or explicit policy gate" + }, + { + "evidence_id": "local-organizer-stale-delete", + "claim_id": "stale_correction_delete_reviewable", + "requirement": "cite", + "quote": "scored before promotion" + } + ], + "negative_traps": [ + { + "trap_id": "local-organizer-authority-write-decoy", + "type": "stale_fact", + "evidence_ids": [ + "local-organizer-decoy" + ], + "failure_if_used": true + } + ], + "scoring_rubric": { + "dimensions": { + "answer_correctness": { + "weight": 0.2, + "max_points": 1.0, + "criteria": { + "requires": [ + "proposal_only", + "strong_gate_escalation", + "stale_correction_delete_reviewable" + ] + } + }, + "evidence_grounding": { + "weight": 0.2, + "max_points": 1.0, + "criteria": { + "requires_source_refs": true + } + }, + "lifecycle_behavior": { + "weight": 0.2, + "max_points": 1.0, + "criteria": { + "stale_correction_delete_score": 1.0 + } + }, + "source_immutability": { + "weight": 0.2, + "max_points": 1.0, + "criteria": { + "max_source_mutations": 0, + "max_silent_memory_authority_mutations": 0 + } + }, + "latency_resource": { + "weight": 0.2, + "max_points": 1.0, + "criteria": { + "max_latency_ms": 5000.0, + "requires_resource_footprint": true + } + } + }, + "pass_threshold": 1.0, + "hard_fail_rules": [ + "source mutation count must remain zero", + "silent Memory Authority mutation count must remain zero", + "strong model or explicit policy gate required for authority-changing work" + ] + }, + "allowed_uncertainty": { + "can_answer_unknown": false, + "acceptable_phrases": [], + "fallback_action": "block_or_escalate" + }, + "tags": [ + "r1_model_ladder", + "local_background_organizer", + "proposal_only", + "fixture_backed" + ] +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts.rs index c50ccb95..9cfdfabe 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts.rs @@ -2,6 +2,7 @@ mod answer; mod consolidation; mod cost; mod knowledge; +mod local_organizer; mod memory; mod proactive; mod recovery; @@ -13,6 +14,10 @@ pub(super) use self::{ consolidation::{ConsolidationFixture, ConsolidationProposalFixture}, cost::CostReport, knowledge::{DerivedPageArtifact, DerivedPageRebuild, DerivedPageSection}, + local_organizer::{ + LocalOrganizerFixture, LocalOrganizerJobReport, LocalOrganizerSummaryReport, + LocalOrganizerTierReport, + }, memory::{MemorySummaryArtifact, MemorySummaryEntry, MemorySummarySourceTrace}, proactive::{ProactiveBriefArtifact, ProactiveSuggestion}, recovery::{ diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs new file mode 100644 index 00000000..711e9728 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs @@ -0,0 +1,85 @@ +use crate::{CostReport, Deserialize, Serialize, Value}; + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct LocalOrganizerFixture { + pub(crate) model_tier: String, + pub(crate) model_profile_id: String, + pub(crate) runtime_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) runtime_commit: Option, + #[serde(default)] + pub(crate) reproducibility_provenance: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) extraction_f1: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) extraction_f1_blocker: Option, + pub(crate) json_schema_valid: bool, + pub(crate) citation_source_ref_coverage: f64, + pub(crate) unsupported_claim_rate: f64, + pub(crate) stale_correction_delete_score: f64, + #[serde(default)] + pub(crate) source_mutation_count: usize, + #[serde(default)] + pub(crate) silent_memory_authority_mutation_count: usize, + #[serde(default)] + pub(crate) required_escalation_count: usize, + #[serde(default)] + pub(crate) completed_escalation_count: usize, + #[serde(default)] + pub(crate) resource_footprint: Value, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct LocalOrganizerJobReport { + pub(crate) model_tier: String, + pub(crate) model_profile_id: String, + pub(crate) runtime_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) runtime_commit: Option, + pub(crate) reproducibility_provenance: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) extraction_f1: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) extraction_f1_blocker: Option, + pub(crate) json_schema_valid: bool, + pub(crate) citation_source_ref_coverage: f64, + pub(crate) unsupported_claim_rate: f64, + pub(crate) stale_correction_delete_score: f64, + pub(crate) source_mutation_count: usize, + pub(crate) silent_memory_authority_mutation_count: usize, + pub(crate) required_escalation_count: usize, + pub(crate) completed_escalation_count: usize, + pub(crate) escalation_rate: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) latency_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) cost: Option, + pub(crate) resource_footprint: Value, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub(crate) struct LocalOrganizerSummaryReport { + pub(crate) job_count: usize, + pub(crate) extraction_f1_not_encoded_count: usize, + pub(crate) json_schema_valid_count: usize, + pub(crate) citation_source_ref_coverage: f64, + pub(crate) unsupported_claim_rate: f64, + pub(crate) stale_correction_delete_score: f64, + pub(crate) source_mutation_count: usize, + pub(crate) silent_memory_authority_mutation_count: usize, + pub(crate) escalation_rate: f64, + pub(crate) mean_latency_ms: Option, + pub(crate) p95_latency_ms: Option, + pub(crate) tiers: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct LocalOrganizerTierReport { + pub(crate) model_tier: String, + pub(crate) job_count: usize, + pub(crate) mean_latency_ms: Option, + pub(crate) p95_latency_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) total_cost: Option, + pub(crate) resource_footprints: Vec, +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/cli.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/cli.rs index bb00571f..2e568bc1 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/cli.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/cli.rs @@ -76,6 +76,13 @@ pub(super) struct ValidateSourceBackedQualityArgs { pub(super) report: PathBuf, } +#[derive(Debug, Parser)] +pub(super) struct ValidateLocalOrganizerArgs { + /// Generated real_world_job JSON report to gate. + #[arg(long, value_name = "FILE", default_value = DEFAULT_REPORT_PATH)] + pub(super) report: PathBuf, +} + #[derive(Debug, Parser)] pub(super) struct ExportQuantitativeProductManifestArgs { /// Generated real_world_job JSON report to export. @@ -143,6 +150,8 @@ pub(super) enum Command { Run(RunArgs), /// Render Markdown from a generated real_world_job JSON report. Publish(PublishArgs), + /// Fail unless the R1 local background organizer benchmark gate passes. + ValidateLocalOrganizer(ValidateLocalOrganizerArgs), /// Fail unless the generated source-backed quality benchmark gate passes. ValidateSourceBackedQuality(ValidateSourceBackedQualityArgs), } diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/commands.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/commands.rs index c1927ec6..038e2e44 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/commands.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/commands.rs @@ -2,11 +2,11 @@ use crate::{ AdapterReport, BTreeSet, CaptureIntegrationReport, CorpusProfile, ExportQuantitativeAuditManifestArgs, ExportQuantitativeProductManifestArgs, OffsetDateTime, Path, PathBuf, PrivateCorpusRedaction, PublishArgs, QuantitativeReportInput, REPORT_SCHEMA, - RealWorldJob, RealWorldReport, Result, Rfc3339, RunArgs, TypedStatus, VERSION, - ValidateSourceBackedQualityArgs, eyre, fs, + RealWorldJob, RealWorldReport, Rfc3339, RunArgs, TypedStatus, VERSION, + ValidateLocalOrganizerArgs, ValidateSourceBackedQualityArgs, eyre, fs, }; -pub(super) fn run_command(args: RunArgs) -> Result<()> { +pub(super) fn run_command(args: RunArgs) -> crate::Result<()> { let jobs = load_jobs(&args.fixtures)?; let report = build_report(&jobs, &args)?; let json = serde_json::to_string_pretty(&report)?; @@ -14,7 +14,7 @@ pub(super) fn run_command(args: RunArgs) -> Result<()> { write_or_print(args.out.as_deref(), json.as_str()) } -pub(super) fn publish_command(args: PublishArgs) -> Result<()> { +pub(super) fn publish_command(args: PublishArgs) -> crate::Result<()> { let raw = fs::read_to_string(&args.report)?; let report = serde_json::from_str::(&raw)?; let markdown = crate::render_markdown(&report, &args.report); @@ -24,7 +24,7 @@ pub(super) fn publish_command(args: PublishArgs) -> Result<()> { pub(super) fn validate_source_backed_quality_command( args: ValidateSourceBackedQualityArgs, -) -> Result<()> { +) -> crate::Result<()> { let raw = fs::read_to_string(&args.report)?; let report = serde_json::from_str::(&raw)?; @@ -33,9 +33,19 @@ pub(super) fn validate_source_backed_quality_command( }) } +pub(super) fn validate_local_organizer_command( + args: ValidateLocalOrganizerArgs, +) -> crate::Result<()> { + let raw = fs::read_to_string(&args.report)?; + let report = serde_json::from_str::(&raw)?; + + validate_local_organizer_gate(&report) + .map_err(|failures| eyre::eyre!("local organizer gate failed: {}", failures.join(", "))) +} + pub(super) fn export_quantitative_product_manifest_command( args: ExportQuantitativeProductManifestArgs, -) -> Result<()> { +) -> crate::Result<()> { let raw = fs::read_to_string(&args.report)?; let report = serde_json::from_str::(&raw)?; let manifest = crate::quantitative_product_manifest_from_report(&report, &args)?; @@ -46,7 +56,7 @@ pub(super) fn export_quantitative_product_manifest_command( pub(super) fn export_quantitative_audit_manifest_command( args: ExportQuantitativeAuditManifestArgs, -) -> Result<()> { +) -> crate::Result<()> { let jobs = load_jobs(&args.fixtures)?; let manifest = crate::quantitative_audit_manifest_from_jobs(jobs.as_slice(), &args)?; let json = serde_json::to_string_pretty(&manifest)?; @@ -54,7 +64,7 @@ pub(super) fn export_quantitative_audit_manifest_command( write_or_print(args.out.as_deref(), json.as_str()) } -fn load_jobs(path: &Path) -> Result> { +fn load_jobs(path: &Path) -> crate::Result> { let paths = fixture_paths(path)?; let mut jobs = Vec::with_capacity(paths.len()); @@ -71,7 +81,7 @@ fn load_jobs(path: &Path) -> Result> { Ok(jobs) } -fn fixture_paths(path: &Path) -> Result> { +fn fixture_paths(path: &Path) -> crate::Result> { if path.is_file() { return Ok(vec![path.to_path_buf()]); } @@ -92,7 +102,7 @@ fn fixture_paths(path: &Path) -> Result> { Ok(paths) } -fn collect_fixture_paths(path: &Path, paths: &mut Vec) -> Result<()> { +fn collect_fixture_paths(path: &Path, paths: &mut Vec) -> crate::Result<()> { for entry in fs::read_dir(path)? { let entry = entry?; let entry_path = entry.path(); @@ -107,7 +117,7 @@ fn collect_fixture_paths(path: &Path, paths: &mut Vec) -> Result<()> { Ok(()) } -fn build_report(jobs: &[RealWorldJob], args: &RunArgs) -> Result { +fn build_report(jobs: &[RealWorldJob], args: &RunArgs) -> crate::Result { if jobs.is_empty() { return Err(eyre::eyre!("At least one real_world_job fixture is required.")); } @@ -176,6 +186,100 @@ fn build_report(jobs: &[RealWorldJob], args: &RunArgs) -> Result std::result::Result<(), Vec> { + let Some(summary) = &report.summary.local_organizer else { + return Err(vec!["missing local_organizer summary".to_string()]); + }; + let mut failures = Vec::new(); + + if summary.job_count == 0 { + failures.push("local_organizer job_count must be greater than zero".to_string()); + } + if summary.json_schema_valid_count != summary.job_count { + failures.push("every local organizer output must be valid JSON/schema".to_string()); + } + if summary.citation_source_ref_coverage < 1.0 { + failures.push("citation/source-ref coverage must be 1.0".to_string()); + } + if summary.unsupported_claim_rate > 0.0 { + failures.push("unsupported claim rate must be 0.0".to_string()); + } + if summary.stale_correction_delete_score < 1.0 { + failures.push("stale/correction/delete score must be 1.0".to_string()); + } + if summary.source_mutation_count > 0 { + failures.push("source mutation count must be zero".to_string()); + } + if summary.silent_memory_authority_mutation_count > 0 { + failures.push("silent Memory Authority mutation count must be zero".to_string()); + } + if summary.escalation_rate < 1.0 { + failures.push("required L2 to L3/L4 escalations must complete".to_string()); + } + if summary.mean_latency_ms.is_none() || summary.p95_latency_ms.is_none() { + failures.push("mean and p95 latency must be reported".to_string()); + } + if summary.tiers.is_empty() { + failures.push("cost/resource footprint by model tier must be reported".to_string()); + } + + for tier in &summary.tiers { + if tier.mean_latency_ms.is_none() || tier.p95_latency_ms.is_none() { + failures.push(format!("tier {} must report mean and p95 latency", tier.model_tier)); + } + if tier.total_cost.as_ref().is_none_or(|cost| { + cost.currency.as_deref().is_none_or(str::is_empty) + && cost.amount.is_none() + && cost.input_tokens.is_none() + && cost.output_tokens.is_none() + }) { + failures.push(format!("tier {} must report cost footprint", tier.model_tier)); + } + if tier.resource_footprints.is_empty() { + failures.push(format!("tier {} must report resource footprint", tier.model_tier)); + } + } + for job in &report.jobs { + let Some(local) = &job.local_organizer else { + continue; + }; + + if job.status != TypedStatus::Pass { + failures.push(format!("local organizer job {} did not pass", job.job_id)); + } + if local.model_tier == "L2" && local.required_escalation_count == 0 { + failures.push(format!( + "local organizer job {} must encode at least one required L2 to L3/L4 escalation", + job.job_id + )); + } + if local.completed_escalation_count < local.required_escalation_count { + failures.push(format!( + "local organizer job {} must complete all required L2 to L3/L4 escalations", + job.job_id + )); + } + if local.extraction_f1.is_none() + && local.extraction_f1_blocker.as_deref().is_none_or(str::is_empty) + { + failures.push(format!( + "local organizer job {} must report extraction_f1 or a not-encoded blocker", + job.job_id + )); + } + if local.runtime_commit.as_deref().is_none_or(str::is_empty) + || local.reproducibility_provenance.is_empty() + { + failures.push(format!( + "local organizer job {} must report runtime commit and reproducibility provenance", + job.job_id + )); + } + } + + if failures.is_empty() { Ok(()) } else { Err(failures) } +} + fn corpus_profile(jobs: &[RealWorldJob]) -> String { let profiles = jobs.iter().map(|job| job.corpus.profile.as_str()).collect::>(); @@ -186,7 +290,7 @@ fn corpus_profile(jobs: &[RealWorldJob]) -> String { } } -fn adapter_report(args: &RunArgs) -> Result { +fn adapter_report(args: &RunArgs) -> crate::Result { Ok(AdapterReport { adapter_id: args.adapter_id.clone(), name: args.adapter_name.clone(), @@ -203,7 +307,7 @@ fn adapter_report(args: &RunArgs) -> Result { }) } -fn typed_status_from_arg(raw: &str, flag: &str) -> Result { +fn typed_status_from_arg(raw: &str, flag: &str) -> crate::Result { match raw { "pass" => Ok(TypedStatus::Pass), "wrong_result" => Ok(TypedStatus::WrongResult), @@ -269,7 +373,7 @@ fn private_corpus_redaction(jobs: &[RealWorldJob]) -> PrivateCorpusRedaction { PrivateCorpusRedaction { policy, private_fixture_count } } -fn write_or_print(path: Option<&Path>, content: &str) -> Result<()> { +fn write_or_print(path: Option<&Path>, content: &str) -> crate::Result<()> { if let Some(path) = path { if let Some(parent) = path.parent() && !parent.as_os_str().is_empty() diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/fixtures.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/fixtures.rs index 3e0575fc..6ce4cd06 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/fixtures.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/fixtures.rs @@ -1,7 +1,7 @@ use crate::{ BTreeMap, CaptureIntegrationReport, ConsolidationFixture, CorpusProfile, Deserialize, - EvidenceLink, ExpectedClaim, OperatorDebugEvidence, ProducedAnswer, Serialize, TypedStatus, - Value, + EvidenceLink, ExpectedClaim, LocalOrganizerFixture, OperatorDebugEvidence, ProducedAnswer, + Serialize, TypedStatus, Value, }; #[derive(Debug, Deserialize)] @@ -278,6 +278,7 @@ pub(super) struct AdapterResponse { pub(super) adapter_id: Option, pub(super) answer: ProducedAnswer, pub(super) consolidation: Option, + pub(super) local_organizer: Option, } fn default_relevance_grade() -> f64 { diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs index 713a44ba..e9807287 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports.rs @@ -4,6 +4,7 @@ mod job_report_domain_metrics; mod job_report_evolution; mod job_report_misc; mod job_report_scoring; +mod local_organizer_reports; pub(super) use self::{ consolidation_reports::{ @@ -20,4 +21,5 @@ pub(super) use self::{ job_report_evolution::{EvolutionJobReport, EvolutionSummary}, job_report_misc::{FollowUpReport, PrivateCorpusRedaction}, job_report_scoring::{FailureCounts, JobMetrics, JobScoring, ScoreboardRankedMetrics}, + local_organizer_reports::LocalOrganizerJobReport, }; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_core.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_core.rs index 95326fa8..ac2ce685 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_core.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_core.rs @@ -2,8 +2,9 @@ use crate::{ AuthorityRecoveryDrillArtifact, CostReport, Deserialize, OperatorDebugEvidence, Serialize, TraceExplainability, TypedStatus, job_reports::{ - ConsolidationJobReport, EvolutionJobReport, KnowledgeJobMetrics, MemorySummaryJobMetrics, - ProactiveBriefJobMetrics, ScheduledMemoryJobMetrics, WorkContinuityJobMetrics, + ConsolidationJobReport, EvolutionJobReport, KnowledgeJobMetrics, LocalOrganizerJobReport, + MemorySummaryJobMetrics, ProactiveBriefJobMetrics, ScheduledMemoryJobMetrics, + WorkContinuityJobMetrics, }, }; @@ -84,6 +85,8 @@ pub(crate) struct JobReport { pub(crate) evolution: Option, #[serde(skip_serializing_if = "Option::is_none")] pub(crate) consolidation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) local_organizer: Option, } #[derive(Debug, Deserialize, Serialize)] diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_scoring.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_scoring.rs index f4b95c9f..6dc07f86 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_scoring.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/job_report_scoring.rs @@ -2,8 +2,8 @@ use crate::{ TypedStatus, job_reports::{ ConsolidationJobReport, DimensionScoreReport, EvolutionJobReport, KnowledgeJobMetrics, - MemorySummaryJobMetrics, ProactiveBriefJobMetrics, ScheduledMemoryJobMetrics, - UnsupportedClaimReport, WorkContinuityJobMetrics, + LocalOrganizerJobReport, MemorySummaryJobMetrics, ProactiveBriefJobMetrics, + ScheduledMemoryJobMetrics, UnsupportedClaimReport, WorkContinuityJobMetrics, }, }; @@ -20,6 +20,7 @@ pub(crate) struct JobScoring { pub(crate) reason: String, pub(crate) evolution: Option, pub(crate) consolidation: Option, + pub(crate) local_organizer: Option, pub(crate) memory_summary: Option, pub(crate) proactive_brief: Option, pub(crate) scheduled_memory: Option, @@ -84,6 +85,12 @@ pub(crate) struct FailureCounts { pub(crate) missed_stale_findings: usize, pub(crate) rebuild_failures: usize, pub(crate) page_usefulness_failures: usize, + pub(crate) local_organizer_schema_failures: usize, + pub(crate) local_organizer_provenance_failures: usize, + pub(crate) local_organizer_unsupported_claims: usize, + pub(crate) local_organizer_lifecycle_failures: usize, + pub(crate) local_organizer_authority_mutations: usize, + pub(crate) local_organizer_escalation_failures: usize, } #[derive(Debug, Default)] diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/local_organizer_reports.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/local_organizer_reports.rs new file mode 100644 index 00000000..bee40f09 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/job_reports/local_organizer_reports.rs @@ -0,0 +1 @@ +pub(crate) use crate::LocalOrganizerJobReport; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/main.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/main.rs index 918c9988..565ba87a 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/main.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/main.rs @@ -44,18 +44,20 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use artifacts::{ AuthorityRecordCount, AuthorityRecoveryDrillArtifact, ConsolidationFixture, ConsolidationProposalFixture, CostReport, DerivedPageArtifact, DerivedPageRebuild, - DerivedPageSection, MemorySummaryArtifact, MemorySummaryEntry, MemorySummarySourceTrace, - ProactiveBriefArtifact, ProactiveSuggestion, ProducedAnswer, ProducedClaim, RecoveryBackupPitr, - RecoveryDeadLetterHandling, RecoveryDegradedRead, RecoveryDrillTopology, RecoveryMeasurement, - RecoveryMigrationRepair, RecoveryOutboxReplay, RecoveryQdrantRebuild, - ScheduledMemoryExecutionTrace, ScheduledMemoryOutput, ScheduledMemoryTaskArtifact, - WorkContinuityObserved, WorkJournalEntryArtifact, WorkJournalJanitorCandidateArtifact, - WorkJournalNextStepArtifact, WorkJournalReadbackArtifact, WorkJournalRejectedOptionArtifact, + DerivedPageSection, LocalOrganizerFixture, LocalOrganizerJobReport, + LocalOrganizerSummaryReport, LocalOrganizerTierReport, MemorySummaryArtifact, + MemorySummaryEntry, MemorySummarySourceTrace, ProactiveBriefArtifact, ProactiveSuggestion, + ProducedAnswer, ProducedClaim, RecoveryBackupPitr, RecoveryDeadLetterHandling, + RecoveryDegradedRead, RecoveryDrillTopology, RecoveryMeasurement, RecoveryMigrationRepair, + RecoveryOutboxReplay, RecoveryQdrantRebuild, ScheduledMemoryExecutionTrace, + ScheduledMemoryOutput, ScheduledMemoryTaskArtifact, WorkContinuityObserved, + WorkJournalEntryArtifact, WorkJournalJanitorCandidateArtifact, WorkJournalNextStepArtifact, + WorkJournalReadbackArtifact, WorkJournalRejectedOptionArtifact, WorkJournalWhereStoppedArtifact, }; use cli::{ Args, Command, ExportQuantitativeAuditManifestArgs, ExportQuantitativeProductManifestArgs, - PublishArgs, RunArgs, ValidateSourceBackedQualityArgs, + PublishArgs, RunArgs, ValidateLocalOrganizerArgs, ValidateSourceBackedQualityArgs, }; use diagnostic_reports::{ OperatorDebugEvidence, OperatorUxGap, TraceExplainability, TraceStageExplainability, @@ -165,6 +167,7 @@ const SUITES: &[&str] = &[ "operator_debugging_ux", "capture_integration", "work_continuity", + "local_background_organizer", "production_ops", "personalization", "core_archival_memory", @@ -194,6 +197,7 @@ fn main() -> Result<()> { commands::export_quantitative_product_manifest_command(args), Command::Run(args) => commands::run_command(args), Command::Publish(args) => commands::publish_command(args), + Command::ValidateLocalOrganizer(args) => commands::validate_local_organizer_command(args), Command::ValidateSourceBackedQuality(args) => commands::validate_source_backed_quality_command(args), } diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown.rs index 5bf557be..e2ea36a6 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown.rs @@ -45,6 +45,7 @@ pub(super) fn render_markdown(report: &RealWorldReport, report_path: &Path) -> S self::evolution::render_markdown_evolution(&mut out, report); self::trace::render_markdown_trace_explainability(&mut out, report); self::domain_metrics::render_markdown_consolidation(&mut out, report); + self::domain_metrics::render_markdown_local_organizer(&mut out, report); self::domain_metrics::render_markdown_memory_summary(&mut out, report); self::domain_metrics::render_markdown_proactive_brief(&mut out, report); self::domain_metrics::render_markdown_scheduled_memory(&mut out, report); diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs index 89848231..f95860b8 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics.rs @@ -1,5 +1,6 @@ mod consolidation; mod knowledge; +mod local_organizer; mod memory; mod proactive; mod scheduled; @@ -7,6 +8,7 @@ mod work; pub(super) use self::{ consolidation::render_markdown_consolidation, knowledge::render_markdown_knowledge, - memory::render_markdown_memory_summary, proactive::render_markdown_proactive_brief, - scheduled::render_markdown_scheduled_memory, work::render_markdown_work_continuity, + local_organizer::render_markdown_local_organizer, memory::render_markdown_memory_summary, + proactive::render_markdown_proactive_brief, scheduled::render_markdown_scheduled_memory, + work::render_markdown_work_continuity, }; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/local_organizer.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/local_organizer.rs new file mode 100644 index 00000000..a3406c5f --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/markdown/domain_metrics/local_organizer.rs @@ -0,0 +1,48 @@ +use crate::markdown::{self, RealWorldReport}; + +pub(in crate::markdown) fn render_markdown_local_organizer( + out: &mut String, + report: &RealWorldReport, +) { + let Some(summary) = &report.summary.local_organizer else { + return; + }; + + out.push_str("## Local Background Organizer\n\n"); + out.push_str("| Jobs | Extraction F1 Not Encoded | Schema Valid | Citation/Source-Ref Coverage | Unsupported Claim Rate | Stale/Correction/Delete | Source Mutations | Silent Memory Authority Mutations | Escalation Rate | Mean Latency | P95 Latency |\n"); + out.push_str( + "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ); + out.push_str(&format!( + "| {} | {} | {} | `{}` | `{}` | `{}` | {} | {} | `{}` | `{}` | `{}` |\n\n", + summary.job_count, + summary.extraction_f1_not_encoded_count, + summary.json_schema_valid_count, + markdown::round3(summary.citation_source_ref_coverage), + markdown::round3(summary.unsupported_claim_rate), + markdown::round3(summary.stale_correction_delete_score), + summary.source_mutation_count, + summary.silent_memory_authority_mutation_count, + markdown::round3(summary.escalation_rate), + markdown::optional_f64(summary.mean_latency_ms, " ms"), + markdown::optional_f64(summary.p95_latency_ms, " ms"), + )); + out.push_str("Local organizer outputs are proposal-only benchmark evidence; passing this slice does not authorize small/local models to write Memory Authority records.\n\n"); + out.push_str("### Model Tiers\n\n"); + out.push_str("| Tier | Jobs | Mean Latency | P95 Latency | Cost | Resource Footprints |\n"); + out.push_str("| --- | ---: | ---: | ---: | --- | --- |\n"); + + for tier in &summary.tiers { + out.push_str(&format!( + "| {} | {} | `{}` | `{}` | {} | {} |\n", + markdown::md_cell(tier.model_tier.as_str()), + tier.job_count, + markdown::optional_f64(tier.mean_latency_ms, " ms"), + markdown::optional_f64(tier.p95_latency_ms, " ms"), + markdown::cost_display(tier.total_cost.as_ref()), + markdown::md_cell(&format!("{} sample(s)", tier.resource_footprints.len())), + )); + } + + out.push('\n'); +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring.rs index 2f0f34a7..6d94b77e 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring.rs @@ -4,6 +4,7 @@ mod consolidation; mod counts; mod dimensions; mod evolution; +mod local_organizer; mod reports; use self::{counts::wrong_result_signal_count, evolution::update_rationale_missing_count}; @@ -11,10 +12,10 @@ use crate::{ BTreeMap, BTreeSet, ConsolidationExecutableGapReport, ConsolidationJobReport, ConsolidationProposalFixture, ConsolidationProposalReport, DimensionScoreReport, EvolutionConflict, EvolutionJobReport, ExpectedClaim, ExpectedEvidenceReport, FailureCounts, - JobMetrics, JobReport, JobScoring, MemoryEvolution, MemorySummaryJobMetrics, NegativeTrap, - ProactiveBriefJobMetrics, ProducedAnswer, RealWorldJob, RequiredEvidence, - RetrievalQualityReport, ScheduledMemoryJobMetrics, TypedStatus, UpdateRationale, Value, - WorkContinuityJobMetrics, + JobMetrics, JobReport, JobScoring, KnowledgeJobMetrics, LocalOrganizerJobReport, + MemoryEvolution, MemorySummaryJobMetrics, NegativeTrap, ProactiveBriefJobMetrics, + ProducedAnswer, RealWorldJob, RequiredEvidence, RetrievalQualityReport, + ScheduledMemoryJobMetrics, TypedStatus, UpdateRationale, Value, WorkContinuityJobMetrics, feature_metrics::{ self, forbidden_diff_key_count, missed_stale_finding_count, page_usefulness_failure_count, }, @@ -23,6 +24,24 @@ use crate::{ summary::{mean_proposal_metric, ratio_or}, }; +struct FailureCountContext<'a> { + job: &'a RealWorldJob, + missing_claims_len: usize, + forbidden_claims_len: usize, + missing_evidence_len: usize, + trap_ids_used_len: usize, + unsupported_claims_len: usize, + latency_violations: usize, + evolution: Option<&'a EvolutionJobReport>, + consolidation: Option<&'a ConsolidationJobReport>, + local_organizer: Option<&'a LocalOrganizerJobReport>, + knowledge: Option<&'a KnowledgeJobMetrics>, + memory_summary: Option<&'a MemorySummaryJobMetrics>, + proactive_brief: Option<&'a ProactiveBriefJobMetrics>, + scheduled_memory: Option<&'a ScheduledMemoryJobMetrics>, + work_continuity: Option<&'a WorkContinuityJobMetrics>, +} + pub(super) fn job_report(job: &RealWorldJob, scoring: JobScoring) -> JobReport { reports::job_report(job, scoring) } @@ -36,6 +55,7 @@ pub(super) fn score_job(job: &RealWorldJob) -> JobScoring { let produced_evidence = self::answers::produced_evidence_ids(answer); let trap_ids_used = self::answers::trap_ids_used(job, &produced_evidence); let consolidation = self::consolidation::consolidation_job_report(job); + let local_organizer = self::local_organizer::local_organizer_job_report(job); if let Some(status) = job.encoding.status { let evolution = self::evolution::evolution_job_report(job, answer, &trap_ids_used, 0); @@ -46,6 +66,7 @@ pub(super) fn score_job(job: &RealWorldJob) -> JobScoring { trap_ids_used, evolution, consolidation, + local_organizer, ); } @@ -64,53 +85,27 @@ pub(super) fn score_job(job: &RealWorldJob) -> JobScoring { unsupported_claims.extend(feature_metrics::unsupported_proactive_suggestions(job, answer)); unsupported_claims.extend(feature_metrics::unsupported_scheduled_outputs(job, answer)); - let operator_counts = self::counts::operator_debug_failure_counts(job); let latency_violations = self::dimensions::latency_violations(job, answer); let hard_fail_hits = self::claims::hard_fail_hits(job, &unsupported_claims, &trap_ids_used); let evolution = self::evolution::evolution_job_report(job, answer, &trap_ids_used, forbidden_claims.len()); - let stale_answers = evolution.as_ref().map_or(0, |report| report.stale_answer_count); - let conflict_detection_missing = evolution - .as_ref() - .map_or(0, |report| report.conflict_count - report.conflict_detection_count); - let update_rationale_missing = evolution.as_ref().map_or(0, update_rationale_missing_count); - let mut counts = FailureCounts { - missing_claims: missing_claims.len(), - forbidden_claims: forbidden_claims.len(), - missing_evidence: missing_evidence.len(), - trap_uses: trap_ids_used.len(), - unsupported_claims: unsupported_claims.len(), - operator_debug_missing: operator_counts.operator_debug_missing, - operator_debug_raw_sql: operator_counts.operator_debug_raw_sql, - operator_debug_trace_gaps: operator_counts.operator_debug_trace_gaps, - operator_debug_repair_unclear: operator_counts.operator_debug_repair_unclear, - stale_answers, - conflict_detection_missing, - update_rationale_missing, + let counts = failure_counts(FailureCountContext { + job, + missing_claims_len: missing_claims.len(), + forbidden_claims_len: forbidden_claims.len(), + missing_evidence_len: missing_evidence.len(), + trap_ids_used_len: trap_ids_used.len(), + unsupported_claims_len: unsupported_claims.len(), latency_violations, - proposal_usefulness_failures: self::consolidation::proposal_usefulness_failures( - consolidation.as_ref(), - ), - lineage_failures: self::consolidation::lineage_failures(consolidation.as_ref()), - review_action_failures: self::consolidation::review_action_failures(consolidation.as_ref()), - source_mutations: consolidation.as_ref().map_or(0, |report| report.source_mutation_count), - blocking_executable_gaps: self::consolidation::blocking_executable_gaps( - consolidation.as_ref(), - ), - untraced_page_sections: knowledge - .as_ref() - .map_or(0, |metrics| metrics.untraced_section_count), - missed_stale_findings: knowledge.as_ref().map_or(0, missed_stale_finding_count), - rebuild_failures: knowledge.as_ref().map_or(0, |metrics| metrics.rebuild_failure_count), - page_usefulness_failures: knowledge.as_ref().map_or(0, page_usefulness_failure_count), - ..FailureCounts::default() - }; - - self::counts::apply_memory_summary_failure_counts(&mut counts, memory_summary.as_ref()); - self::counts::apply_proactive_brief_failure_counts(&mut counts, proactive_brief.as_ref()); - self::counts::apply_scheduled_memory_failure_counts(&mut counts, scheduled_memory.as_ref()); - self::counts::apply_work_continuity_failure_counts(&mut counts, work_continuity.as_ref()); - + evolution: evolution.as_ref(), + consolidation: consolidation.as_ref(), + local_organizer: local_organizer.as_ref(), + knowledge: knowledge.as_ref(), + memory_summary: memory_summary.as_ref(), + proactive_brief: proactive_brief.as_ref(), + scheduled_memory: scheduled_memory.as_ref(), + work_continuity: work_continuity.as_ref(), + }); let dimension_scores = self::dimensions::dimension_scores(job, &counts); let normalized_score = self::dimensions::normalized_score(&dimension_scores); let wrong_result_count = self::counts::wrong_result_count(&counts); @@ -141,9 +136,71 @@ pub(super) fn score_job(job: &RealWorldJob) -> JobScoring { reason, evolution, consolidation, + local_organizer, memory_summary, proactive_brief, scheduled_memory, work_continuity, } } + +fn failure_counts(ctx: FailureCountContext<'_>) -> FailureCounts { + let operator_counts = self::counts::operator_debug_failure_counts(ctx.job); + let stale_answers = ctx.evolution.map_or(0, |report| report.stale_answer_count); + let conflict_detection_missing = + ctx.evolution.map_or(0, |report| report.conflict_count - report.conflict_detection_count); + let update_rationale_missing = ctx.evolution.map_or(0, update_rationale_missing_count); + let mut counts = FailureCounts { + missing_claims: ctx.missing_claims_len, + forbidden_claims: ctx.forbidden_claims_len, + missing_evidence: ctx.missing_evidence_len, + trap_uses: ctx.trap_ids_used_len, + unsupported_claims: ctx.unsupported_claims_len, + operator_debug_missing: operator_counts.operator_debug_missing, + operator_debug_raw_sql: operator_counts.operator_debug_raw_sql, + operator_debug_trace_gaps: operator_counts.operator_debug_trace_gaps, + operator_debug_repair_unclear: operator_counts.operator_debug_repair_unclear, + stale_answers, + conflict_detection_missing, + update_rationale_missing, + latency_violations: ctx.latency_violations, + proposal_usefulness_failures: self::consolidation::proposal_usefulness_failures( + ctx.consolidation, + ), + lineage_failures: self::consolidation::lineage_failures(ctx.consolidation), + review_action_failures: self::consolidation::review_action_failures(ctx.consolidation), + source_mutations: ctx.consolidation.map_or(0, |report| report.source_mutation_count) + + self::local_organizer::authority_mutations(ctx.local_organizer), + blocking_executable_gaps: self::consolidation::blocking_executable_gaps(ctx.consolidation), + local_organizer_schema_failures: self::local_organizer::schema_failures( + ctx.local_organizer, + ), + local_organizer_provenance_failures: self::local_organizer::provenance_failures( + ctx.local_organizer, + ), + local_organizer_unsupported_claims: self::local_organizer::unsupported_claim_failures( + ctx.local_organizer, + ), + local_organizer_lifecycle_failures: self::local_organizer::lifecycle_failures( + ctx.local_organizer, + ), + local_organizer_authority_mutations: self::local_organizer::authority_mutations( + ctx.local_organizer, + ), + local_organizer_escalation_failures: self::local_organizer::escalation_failures( + ctx.local_organizer, + ), + untraced_page_sections: ctx.knowledge.map_or(0, |metrics| metrics.untraced_section_count), + missed_stale_findings: ctx.knowledge.map_or(0, missed_stale_finding_count), + rebuild_failures: ctx.knowledge.map_or(0, |metrics| metrics.rebuild_failure_count), + page_usefulness_failures: ctx.knowledge.map_or(0, page_usefulness_failure_count), + ..FailureCounts::default() + }; + + self::counts::apply_memory_summary_failure_counts(&mut counts, ctx.memory_summary); + self::counts::apply_proactive_brief_failure_counts(&mut counts, ctx.proactive_brief); + self::counts::apply_scheduled_memory_failure_counts(&mut counts, ctx.scheduled_memory); + self::counts::apply_work_continuity_failure_counts(&mut counts, ctx.work_continuity); + + counts +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs index 6d2ae868..1bfcd07e 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/declared.rs @@ -1,6 +1,6 @@ use crate::scoring::{ - ConsolidationJobReport, DimensionScoreReport, EvolutionJobReport, JobScoring, RealWorldJob, - TypedStatus, + ConsolidationJobReport, DimensionScoreReport, EvolutionJobReport, JobScoring, + LocalOrganizerJobReport, RealWorldJob, TypedStatus, }; pub(in crate::scoring) fn score_declared_job( @@ -9,6 +9,7 @@ pub(in crate::scoring) fn score_declared_job( trap_ids_used: Vec, evolution: Option, consolidation: Option, + local_organizer: Option, ) -> JobScoring { JobScoring { status, @@ -26,6 +27,7 @@ pub(in crate::scoring) fn score_declared_job( .unwrap_or_else(|| "Job did not reach a runnable scoring state.".to_string()), evolution, consolidation, + local_organizer, memory_summary: None, proactive_brief: None, scheduled_memory: None, diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs index 57da2dbb..c823953a 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/counts/wrong_results.rs @@ -40,6 +40,7 @@ pub(in crate::scoring) fn wrong_result_count(counts: &FailureCounts) -> usize { + counts.missed_stale_findings + counts.rebuild_failures + counts.page_usefulness_failures + + local_organizer_wrong_result_count(counts) } pub(in crate::scoring) fn wrong_result_signal_count(counts: &FailureCounts) -> usize { @@ -82,6 +83,7 @@ pub(in crate::scoring) fn wrong_result_signal_count(counts: &FailureCounts) -> u + counts.missed_stale_findings + counts.rebuild_failures + counts.page_usefulness_failures + + local_organizer_wrong_result_count(counts) } fn work_continuity_wrong_result_count(counts: &FailureCounts) -> usize { @@ -99,3 +101,11 @@ fn work_continuity_wrong_result_count(counts: &FailureCounts) -> usize { + counts.work_continuity_janitor_false_promotion + counts.work_continuity_journal_only_authority_claim } + +fn local_organizer_wrong_result_count(counts: &FailureCounts) -> usize { + counts.local_organizer_schema_failures + + counts.local_organizer_provenance_failures + + counts.local_organizer_unsupported_claims + + counts.local_organizer_lifecycle_failures + + counts.local_organizer_escalation_failures +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/dimensions.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/dimensions.rs index 6c0cd2ca..d89d5462 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/dimensions.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/dimensions.rs @@ -123,7 +123,10 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.work_continuity_inferred_step_as_instruction > 0 || counts.work_continuity_janitor_false_promotion > 0 || counts.work_continuity_journal_only_authority_claim > 0 - || counts.page_usefulness_failures > 0, + || counts.page_usefulness_failures > 0 + || counts.local_organizer_schema_failures > 0 + || counts.local_organizer_lifecycle_failures > 0 + || counts.local_organizer_escalation_failures > 0, "evidence_grounding" => counts.missing_evidence > 0 || counts.unsupported_claims > 0 @@ -136,7 +139,8 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.work_continuity_handoff_source_ref_missing > 0 || counts.work_continuity_redaction_missing > 0 || counts.work_continuity_sensitive_marker_persistence > 0 - || counts.untraced_page_sections > 0, + || counts.untraced_page_sections > 0 + || counts.local_organizer_provenance_failures > 0, "trap_avoidance" => counts.trap_uses > 0 || counts.memory_summary_invalid_current_entries > 0 @@ -146,13 +150,15 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.scheduled_memory_tombstone_violations > 0 || counts.work_continuity_rejected_option_resurrection > 0 || counts.work_continuity_sensitive_marker_persistence > 0 - || counts.missed_stale_findings > 0, + || counts.missed_stale_findings > 0 + || counts.local_organizer_authority_mutations > 0, "uncertainty_handling" => counts.unsupported_claims > 0 || counts.memory_summary_unsupported_current_entries > 0 || counts.proactive_brief_unsupported_current_suggestions > 0 || counts.scheduled_memory_unsupported_current_outputs > 0 - || counts.work_continuity_journal_only_authority_claim > 0, + || counts.work_continuity_journal_only_authority_claim > 0 + || counts.local_organizer_unsupported_claims > 0, "lifecycle_behavior" => counts.stale_answers > 0 || counts.conflict_detection_missing > 0 @@ -177,8 +183,11 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.work_continuity_inferred_step_as_instruction > 0 || counts.work_continuity_janitor_false_promotion > 0 || counts.work_continuity_journal_only_authority_claim > 0 - || counts.rebuild_failures > 0, - "source_immutability" => counts.source_mutations > 0, + || counts.rebuild_failures > 0 + || counts.local_organizer_lifecycle_failures > 0 + || counts.local_organizer_authority_mutations > 0, + "source_immutability" => + counts.source_mutations > 0 || counts.local_organizer_authority_mutations > 0, "proposal_usefulness" => counts.proposal_usefulness_failures > 0, "lineage_completeness" => counts.lineage_failures > 0, "review_action_correctness" => counts.review_action_failures > 0, @@ -189,7 +198,8 @@ fn dimension_score(dimension_id: &str, max_points: f64, counts: &FailureCounts) || counts.operator_debug_raw_sql > 0 || counts.operator_debug_trace_gaps > 0 || counts.scheduled_memory_missing_trace > 0 - || counts.work_continuity_reset_resume_missing > 0, + || counts.work_continuity_reset_resume_missing > 0 + || counts.local_organizer_schema_failures > 0, "trace_readback" => counts.scheduled_memory_missing_trace > 0, "latency_resource" => counts.latency_violations > 0, "personalization_fit" | "ownership_correctness" => diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/local_organizer.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/local_organizer.rs new file mode 100644 index 00000000..f868214d --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/local_organizer.rs @@ -0,0 +1,69 @@ +use crate::{ + LocalOrganizerJobReport, RealWorldJob, formatting, + scoring::{self, answers}, +}; + +pub(super) fn local_organizer_job_report(job: &RealWorldJob) -> Option { + let answer = answers::produced_answer(job); + let fixture = job.corpus.adapter_response.as_ref()?.local_organizer.as_ref()?; + let escalation_rate = scoring::ratio_or( + fixture.completed_escalation_count, + fixture.required_escalation_count, + 1.0, + ); + + Some(LocalOrganizerJobReport { + model_tier: fixture.model_tier.clone(), + model_profile_id: fixture.model_profile_id.clone(), + runtime_id: fixture.runtime_id.clone(), + runtime_commit: fixture.runtime_commit.clone(), + reproducibility_provenance: fixture.reproducibility_provenance.clone(), + extraction_f1: fixture.extraction_f1.map(crate::scoring::round3), + extraction_f1_blocker: fixture.extraction_f1_blocker.clone(), + json_schema_valid: fixture.json_schema_valid, + citation_source_ref_coverage: formatting::round3(fixture.citation_source_ref_coverage), + unsupported_claim_rate: formatting::round3(fixture.unsupported_claim_rate), + stale_correction_delete_score: formatting::round3(fixture.stale_correction_delete_score), + source_mutation_count: fixture.source_mutation_count, + silent_memory_authority_mutation_count: fixture.silent_memory_authority_mutation_count, + required_escalation_count: fixture.required_escalation_count, + completed_escalation_count: fixture.completed_escalation_count, + escalation_rate, + latency_ms: answer.latency_ms.map(crate::scoring::round3), + cost: answer.cost.clone(), + resource_footprint: fixture.resource_footprint.clone(), + }) +} + +pub(super) fn schema_failures(local_organizer: Option<&LocalOrganizerJobReport>) -> usize { + local_organizer.map_or(0, |report| usize::from(!report.json_schema_valid)) +} + +pub(super) fn provenance_failures(local_organizer: Option<&LocalOrganizerJobReport>) -> usize { + local_organizer.map_or(0, |report| usize::from(report.citation_source_ref_coverage < 1.0)) +} + +pub(super) fn unsupported_claim_failures( + local_organizer: Option<&LocalOrganizerJobReport>, +) -> usize { + local_organizer.map_or(0, |report| usize::from(report.unsupported_claim_rate > 0.0)) +} + +pub(super) fn lifecycle_failures(local_organizer: Option<&LocalOrganizerJobReport>) -> usize { + local_organizer.map_or(0, |report| usize::from(report.stale_correction_delete_score < 1.0)) +} + +pub(super) fn authority_mutations(local_organizer: Option<&LocalOrganizerJobReport>) -> usize { + local_organizer.map_or(0, |report| { + report.source_mutation_count + report.silent_memory_authority_mutation_count + }) +} + +pub(super) fn escalation_failures(local_organizer: Option<&LocalOrganizerJobReport>) -> usize { + local_organizer.map_or(0, |report| { + usize::from( + report.completed_escalation_count < report.required_escalation_count + || (report.model_tier == "L2" && report.required_escalation_count == 0), + ) + }) +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/reports.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/reports.rs index 94816348..608a4c98 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/reports.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/scoring/reports.rs @@ -73,6 +73,7 @@ pub(super) fn job_report(job: &RealWorldJob, scoring: JobScoring) -> JobReport { operator_debug: job.operator_debug.clone(), evolution: scoring.evolution, consolidation: scoring.consolidation, + local_organizer: scoring.local_organizer, } } diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/summary.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/summary.rs index 64d2b03f..2d70b9d6 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/summary.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/summary.rs @@ -1,5 +1,6 @@ mod consolidation; mod knowledge; +mod local_organizer; mod memory; mod metrics; mod proactive; @@ -10,9 +11,9 @@ mod work; use crate::{ ConsolidationSummaryReport, CostReport, EvolutionSummary, FollowUpReport, JobReport, - KnowledgeSummary, MemorySummaryReport, ProactiveBriefSummaryReport, RealWorldJob, - ReportSummary, ScheduledMemorySummaryReport, SuiteReport, TypedStatus, - WorkContinuitySummaryReport, + KnowledgeSummary, LocalOrganizerSummaryReport, MemorySummaryReport, + ProactiveBriefSummaryReport, RealWorldJob, ReportSummary, ScheduledMemorySummaryReport, + SuiteReport, TypedStatus, WorkContinuitySummaryReport, }; pub(super) fn suite_reports(jobs: &[JobReport]) -> Vec { @@ -87,6 +88,10 @@ fn consolidation_summary(jobs: &[JobReport]) -> ConsolidationSummaryReport { consolidation::consolidation_summary_impl(jobs) } +fn local_organizer_summary(jobs: &[JobReport]) -> Option { + local_organizer::local_organizer_summary_impl(jobs) +} + fn memory_summary_summary(jobs: &[JobReport]) -> Option { memory::memory_summary_summary_impl(jobs) } diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/summary/local_organizer.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/summary/local_organizer.rs new file mode 100644 index 00000000..9ebc49ef --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/summary/local_organizer.rs @@ -0,0 +1,127 @@ +use std::collections::BTreeMap; + +use crate::{ + CostReport, JobReport, LocalOrganizerJobReport, LocalOrganizerSummaryReport, + LocalOrganizerTierReport, Value, formatting, + summary::{self}, +}; + +pub(super) fn local_organizer_summary_impl( + jobs: &[JobReport], +) -> Option { + let reports = jobs.iter().filter_map(|job| job.local_organizer.as_ref()).collect::>(); + + if reports.is_empty() { + return None; + } + + let source_mutation_count = reports.iter().map(|report| report.source_mutation_count).sum(); + let silent_memory_authority_mutation_count = + reports.iter().map(|report| report.silent_memory_authority_mutation_count).sum(); + let citation_source_ref_coverage = mean_metric( + reports.iter().map(|report| report.citation_source_ref_coverage).collect::>(), + ); + let unsupported_claim_rate = + mean_metric(reports.iter().map(|report| report.unsupported_claim_rate).collect::>()); + let stale_correction_delete_score = mean_metric( + reports.iter().map(|report| report.stale_correction_delete_score).collect::>(), + ); + let escalation_rate = + mean_metric(reports.iter().map(|report| report.escalation_rate).collect::>()); + let latencies = reports.iter().filter_map(|report| report.latency_ms).collect::>(); + let extraction_f1_not_encoded_count = + reports.iter().filter(|report| report.extraction_f1.is_none()).count(); + let json_schema_valid_count = reports.iter().filter(|report| report.json_schema_valid).count(); + + Some(LocalOrganizerSummaryReport { + job_count: reports.len(), + extraction_f1_not_encoded_count, + json_schema_valid_count, + citation_source_ref_coverage, + unsupported_claim_rate, + stale_correction_delete_score, + source_mutation_count, + silent_memory_authority_mutation_count, + escalation_rate, + mean_latency_ms: summary::mean_latency_for_values(latencies.as_slice()), + p95_latency_ms: percentile_latency(latencies, 0.95), + tiers: tier_reports(&reports), + }) +} + +fn tier_reports(reports: &[&LocalOrganizerJobReport]) -> Vec { + let mut by_tier = BTreeMap::>::new(); + + for report in reports { + by_tier.entry(report.model_tier.clone()).or_default().push(*report); + } + + by_tier + .into_iter() + .map(|(model_tier, reports)| { + let latencies = + reports.iter().filter_map(|report| report.latency_ms).collect::>(); + let costs = + reports.iter().filter_map(|report| report.cost.as_ref()).collect::>(); + let resource_footprints = reports + .iter() + .map(|report| report.resource_footprint.clone()) + .filter(|footprint| !footprint.is_null()) + .collect::>(); + + LocalOrganizerTierReport { + model_tier, + job_count: reports.len(), + mean_latency_ms: summary::mean_latency_for_values(latencies.as_slice()), + p95_latency_ms: percentile_latency(latencies, 0.95), + total_cost: total_cost_for_values(costs.as_slice()), + resource_footprints, + } + }) + .collect() +} + +fn mean_metric(values: Vec) -> f64 { + if values.is_empty() { + 0.0 + } else { + formatting::round3(values.iter().sum::() / values.len() as f64) + } +} + +fn percentile_latency(mut values: Vec, percentile: f64) -> Option { + if values.is_empty() { + return None; + } + + values.sort_by(|left, right| left.total_cmp(right)); + + let index = ((values.len() as f64 - 1.0) * percentile).ceil() as usize; + + values.get(index).copied().map(formatting::round3) +} + +fn total_cost_for_values(costs: &[&CostReport]) -> Option { + if costs.is_empty() { + return None; + } + + let currency = costs.iter().find_map(|cost| cost.currency.clone()); + let amount = sum_optional_f64(costs.iter().filter_map(|cost| cost.amount)); + let input_tokens = sum_optional_u64(costs.iter().filter_map(|cost| cost.input_tokens)); + let output_tokens = sum_optional_u64(costs.iter().filter_map(|cost| cost.output_tokens)); + + Some(CostReport { currency, amount, input_tokens, output_tokens }) +} + +fn sum_optional_f64(values: impl Iterator) -> Option { + let values = values.collect::>(); + + if values.is_empty() { None } else { Some(formatting::round3(values.iter().sum())) } +} + +fn sum_optional_u64(values: impl Iterator) -> Option { + let values = values.collect::>(); + + if values.is_empty() { None } else { Some(values.iter().sum()) } +} diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/summary/report.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/summary/report.rs index 222b6b88..3de8bbe5 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/summary/report.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/summary/report.rs @@ -99,6 +99,7 @@ pub(super) fn report_summary_impl(jobs: &[JobReport], suites: &[SuiteReport]) -> .map(|debug| debug.ux_gaps.len()) .sum(), consolidation: summary::consolidation_summary(jobs), + local_organizer: summary::local_organizer_summary(jobs), memory_summary: summary::memory_summary_summary(jobs), proactive_brief: summary::proactive_brief_summary(jobs), scheduled_memory: summary::scheduled_memory_summary(jobs), diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/summary_reports/summary_report_core.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/summary_reports/summary_report_core.rs index dea563fa..26dfa8cc 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/summary_reports/summary_report_core.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/summary_reports/summary_report_core.rs @@ -1,5 +1,5 @@ use crate::{ - CostReport, Deserialize, Serialize, + CostReport, Deserialize, LocalOrganizerSummaryReport, Serialize, summary_reports::{ KnowledgeSummary, MemorySummaryReport, ProactiveBriefSummaryReport, ScheduledMemorySummaryReport, WorkContinuitySummaryReport, @@ -84,6 +84,8 @@ pub(crate) struct ReportSummary { #[serde(default)] pub(crate) consolidation: ConsolidationSummaryReport, #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) local_organizer: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub(crate) memory_summary: Option, #[serde(skip_serializing_if = "Option::is_none")] pub(crate) proactive_brief: Option, diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation.rs index a725058b..dc1209db 100644 --- a/apps/elf-eval/src/bin/real_world_job_benchmark/validation.rs +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation.rs @@ -4,6 +4,7 @@ mod common; mod consolidation; mod expectations; mod job_rules; +mod local_organizer; mod memory_summary; mod page; mod proactive; @@ -67,6 +68,7 @@ pub(super) fn validate_job(job: &RealWorldJob, path: &Path) -> Result<()> { self::basics::validate_expected_answer(job, path)?; self::basics::validate_required_evidence(job, path)?; self::consolidation::validate_consolidation_fixture(job, path)?; + self::local_organizer::validate_local_organizer_fixture(job, path)?; self::adapter::validate_adapter_response(job, path)?; self::job_rules::validate_scoring_rubric(job, path)?; self::job_rules::validate_allowed_uncertainty(job, path)?; diff --git a/apps/elf-eval/src/bin/real_world_job_benchmark/validation/local_organizer.rs b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/local_organizer.rs new file mode 100644 index 00000000..5f2a24e0 --- /dev/null +++ b/apps/elf-eval/src/bin/real_world_job_benchmark/validation/local_organizer.rs @@ -0,0 +1,71 @@ +use crate::validation::{Path, RealWorldJob, Result, eyre}; + +pub(super) fn validate_local_organizer_fixture(job: &RealWorldJob, path: &Path) -> Result<()> { + let local_organizer = + job.corpus.adapter_response.as_ref().and_then(|response| response.local_organizer.as_ref()); + + if job.suite == "local_background_organizer" + && local_organizer.is_none() + && job.encoding.status.is_none() + { + return Err(eyre::eyre!( + "{} local_background_organizer jobs must provide adapter_response.local_organizer.", + path.display() + )); + } + + let Some(local_organizer) = local_organizer else { + return Ok(()); + }; + + if local_organizer.model_tier.trim().is_empty() + || local_organizer.model_profile_id.trim().is_empty() + || local_organizer.runtime_id.trim().is_empty() + { + return Err(eyre::eyre!( + "{} has an incomplete local organizer model/runtime identity.", + path.display() + )); + } + if local_organizer.extraction_f1.is_none() + && local_organizer.extraction_f1_blocker.as_deref().is_none_or(str::is_empty) + { + return Err(eyre::eyre!( + "{} local organizer fixture must report extraction_f1 or extraction_f1_blocker.", + path.display() + )); + } + + for (field, value) in [ + ("extraction_f1", local_organizer.extraction_f1), + ("citation_source_ref_coverage", Some(local_organizer.citation_source_ref_coverage)), + ("unsupported_claim_rate", Some(local_organizer.unsupported_claim_rate)), + ("stale_correction_delete_score", Some(local_organizer.stale_correction_delete_score)), + ] { + let Some(value) = value else { + continue; + }; + + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + return Err(eyre::eyre!( + "{} local organizer metric {field} must be finite and between 0.0 and 1.0.", + path.display() + )); + } + } + + if local_organizer.completed_escalation_count > local_organizer.required_escalation_count { + return Err(eyre::eyre!( + "{} local organizer completed_escalation_count cannot exceed required_escalation_count.", + path.display() + )); + } + if !local_organizer.resource_footprint.is_object() { + return Err(eyre::eyre!( + "{} local organizer resource_footprint must be a JSON object.", + path.display() + )); + } + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark.rs b/apps/elf-eval/tests/real_world_job_benchmark.rs index fe347402..45482e3f 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark.rs @@ -13,6 +13,7 @@ #[path = "real_world_job_benchmark/dreaming_reports.rs"] mod dreaming_reports; #[path = "real_world_job_benchmark/external_adapters.rs"] mod external_adapters; #[path = "real_world_job_benchmark/live_adapter_tasks.rs"] mod live_adapter_tasks; +#[path = "real_world_job_benchmark/local_organizer.rs"] mod local_organizer; #[path = "real_world_job_benchmark/markdown_rendering.rs"] mod markdown_rendering; #[path = "real_world_job_benchmark/memory_evolution.rs"] mod memory_evolution; #[path = "real_world_job_benchmark/memory_summary.rs"] mod memory_summary; diff --git a/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_final_source_backed.rs b/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_final_source_backed.rs index 42eea78c..b74f36ec 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_final_source_backed.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_final_source_backed.rs @@ -53,7 +53,7 @@ fn final_source_backed_closeout_report_preserves_claim_boundaries_and_docs_links assert!(index.contains("2026-07-03-final-source-backed-project-memory-closeout-report.md")); assert!(readme.contains("Final Source-Backed Project Memory Closeout Report - July 3, 2026")); - assert!(readme.contains("Latest real-world benchmark report: July 3, 2026")); + assert!(readme.contains("Latest real-world benchmark report: July 4, 2026")); Ok(()) } diff --git a/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_fixtures/runner.rs b/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_fixtures/runner.rs index e1d852f9..0aab285e 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_fixtures/runner.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_fixtures/runner.rs @@ -7,7 +7,7 @@ use crate::support; fn runner_discovers_nested_fixture_layout() -> Result<()> { let report = support::run_json_report_from(support::fixture_root())?; - assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(83)); + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(84)); Ok(()) } diff --git a/apps/elf-eval/tests/real_world_job_benchmark/local_organizer.rs b/apps/elf-eval/tests/real_world_job_benchmark/local_organizer.rs new file mode 100644 index 00000000..bcd6ae32 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/local_organizer.rs @@ -0,0 +1,160 @@ +use std::{ + env, fs, + process::{self, Command, Output}, +}; + +use color_eyre::Result; +use serde_json::Value; + +use crate::support; + +#[test] +fn r1_local_organizer_report_emits_gate_metrics() -> Result<()> { + let report = support::run_json_report_from( + support::real_world_memory_fixture_dir().join("local_background_organizer"), + )?; + let summary = + report.pointer("/summary/local_organizer").expect("missing local_organizer summary"); + + assert_eq!(summary.pointer("/job_count").and_then(Value::as_u64), Some(1)); + assert_eq!( + summary.pointer("/extraction_f1_not_encoded_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!(summary.pointer("/json_schema_valid_count").and_then(Value::as_u64), Some(1)); + assert_eq!(summary.pointer("/citation_source_ref_coverage").and_then(Value::as_f64), Some(1.0)); + assert_eq!(summary.pointer("/unsupported_claim_rate").and_then(Value::as_f64), Some(0.0)); + assert_eq!( + summary.pointer("/stale_correction_delete_score").and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!(summary.pointer("/source_mutation_count").and_then(Value::as_u64), Some(0)); + assert_eq!( + summary.pointer("/silent_memory_authority_mutation_count").and_then(Value::as_u64), + Some(0) + ); + assert_eq!(summary.pointer("/escalation_rate").and_then(Value::as_f64), Some(1.0)); + assert_eq!(summary.pointer("/mean_latency_ms").and_then(Value::as_f64), Some(42.0)); + assert_eq!(summary.pointer("/p95_latency_ms").and_then(Value::as_f64), Some(42.0)); + + let tiers = support::array_at(summary, "/tiers")?; + + assert_eq!(tiers.len(), 1); + assert_eq!(tiers[0].pointer("/model_tier").and_then(Value::as_str), Some("L2")); + assert_eq!(tiers[0].pointer("/total_cost/input_tokens").and_then(Value::as_u64), Some(320)); + assert!(!support::array_at(&tiers[0], "/resource_footprints")?.is_empty()); + + let job = &support::array_at(&report, "/jobs")?[0]; + + assert_eq!(job.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!( + job.pointer("/local_organizer/extraction_f1_blocker").and_then(Value::as_str), + Some( + "not_encoded: no labeled extraction set is checked in for this first R1 local organizer slice." + ) + ); + assert_eq!( + job.pointer("/local_organizer/runtime_commit").and_then(Value::as_str), + Some("fixture-runtime@2026-07-04") + ); + assert_eq!( + job.pointer("/consolidation/source_mutation_count").and_then(Value::as_u64), + Some(0) + ); + + Ok(()) +} + +#[test] +fn r1_local_organizer_validate_command_accepts_fixture_report() -> Result<()> { + let report = support::run_json_report_from( + support::real_world_memory_fixture_dir().join("local_background_organizer"), + )?; + let output = validate_local_organizer_report(&report, "valid")?; + + assert!( + output.status.success(), + "validate-local-organizer failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + Ok(()) +} + +#[test] +fn r1_local_organizer_validate_command_rejects_missing_l2_escalation() -> Result<()> { + let mut report = support::run_json_report_from( + support::real_world_memory_fixture_dir().join("local_background_organizer"), + )?; + + *report + .pointer_mut("/jobs/0/local_organizer/required_escalation_count") + .expect("missing required_escalation_count") = Value::from(0); + *report + .pointer_mut("/jobs/0/local_organizer/completed_escalation_count") + .expect("missing completed_escalation_count") = Value::from(0); + + let output = validate_local_organizer_report(&report, "missing-escalation")?; + + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must encode at least one required")); + + Ok(()) +} + +#[test] +fn r1_local_organizer_validate_command_rejects_missing_tier_cost() -> Result<()> { + let mut report = support::run_json_report_from( + support::real_world_memory_fixture_dir().join("local_background_organizer"), + )?; + + report + .pointer_mut("/summary/local_organizer/tiers/0") + .expect("missing local organizer tier") + .as_object_mut() + .expect("tier must be object") + .remove("total_cost"); + + let output = validate_local_organizer_report(&report, "missing-tier-cost")?; + + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must report cost footprint")); + + Ok(()) +} + +fn validate_local_organizer_report(report: &Value, suffix: &str) -> Result { + let temp_dir = + env::temp_dir().join(format!("elf-r1-local-organizer-report-{}-{suffix}", process::id())); + + fs::create_dir_all(&temp_dir)?; + + let report_path = temp_dir.join("report.json"); + + fs::write(&report_path, serde_json::to_vec_pretty(&report)?)?; + + let output = Command::new(env!("CARGO_BIN_EXE_real_world_job_benchmark")) + .arg("validate-local-organizer") + .arg("--report") + .arg(&report_path) + .output()?; + + Ok(output) +} + +#[test] +fn r1_local_organizer_task_is_registered() -> Result<()> { + let makefile = + fs::read_to_string(support::workspace_root()?.join("makefiles/benchmark-memory-b.toml"))?; + + for task in [ + "[tasks.real-world-memory-r1-local-organizer]", + "[tasks.real-world-memory-r1-local-organizer-json]", + "[tasks.real-world-memory-r1-local-organizer-validate]", + "[tasks.real-world-memory-r1-local-organizer-report]", + ] { + assert!(makefile.contains(task), "missing cargo make task {task}"); + } + + Ok(()) +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs b/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs index c2d68220..bb53b45d 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/qmd_debug_retest.rs @@ -175,6 +175,6 @@ fn assert_qmd_debug_retest_markdown_and_indexes( ); assert!(readme.contains("qmd Debug-Ergonomics Dreaming Retest Report - June 19, 2026")); assert!(readme.contains("Temporal and Trajectory Adapter Coverage Report - June 23, 2026")); - assert!(readme.contains("Latest real-world benchmark report: July 3, 2026")); + assert!(readme.contains("Latest real-world benchmark report: July 4, 2026")); assert!(readme.contains("keeps the qmd edge unchanged")); } diff --git a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_suites.rs b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_suites.rs index 3a1bc002..6d3cdfc0 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_suites.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_suites.rs @@ -21,6 +21,7 @@ pub(crate) fn assert_root_aggregate_suites(report: &Value) -> Result<()> { "adversarial_quality", "core_archival_memory", "work_continuity", + "local_background_organizer", ] { let suite = support::find_by_field(suites, "/suite_id", suite_id)?; @@ -83,5 +84,11 @@ pub(crate) fn assert_root_aggregate_suites(report: &Value) -> Result<()> { assert_eq!(work_continuity.pointer("/status").and_then(Value::as_str), Some("pass")); assert_eq!(work_continuity.pointer("/encoded_job_count").and_then(Value::as_u64), Some(8)); + let local_organizer = + support::find_by_field(suites, "/suite_id", "local_background_organizer")?; + + assert_eq!(local_organizer.pointer("/status").and_then(Value::as_str), Some("pass")); + assert_eq!(local_organizer.pointer("/encoded_job_count").and_then(Value::as_u64), Some(1)); + Ok(()) } diff --git a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary.rs b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary.rs index dc6657c1..0f70a260 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary.rs @@ -9,6 +9,7 @@ pub(crate) fn assert_root_aggregate_summary(report: &Value) -> Result<()> { root_aggregate_summary_counts::assert_root_summary_counts(report); root_aggregate_summary_scoreboard::assert_root_scoreboard_summary(report)?; root_aggregate_summary_counts::assert_root_consolidation_summary(report); + root_aggregate_summary_counts::assert_root_local_organizer_summary(report); root_aggregate_summary_suite_summaries::assert_root_knowledge_summary(report); root_aggregate_summary_suite_summaries::assert_root_proactive_brief_summary(report); root_aggregate_summary_suite_summaries::assert_root_scheduled_memory_summary(report); diff --git a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_counts.rs b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_counts.rs index 99a123db..2bfefcbd 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_counts.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_counts.rs @@ -1,9 +1,9 @@ use serde_json::Value; pub(crate) fn assert_root_summary_counts(report: &Value) { - assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(83)); - assert_eq!(report.pointer("/summary/encoded_suite_count").and_then(Value::as_u64), Some(19)); - assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(76)); + assert_eq!(report.pointer("/summary/job_count").and_then(Value::as_u64), Some(84)); + assert_eq!(report.pointer("/summary/encoded_suite_count").and_then(Value::as_u64), Some(20)); + assert_eq!(report.pointer("/summary/pass").and_then(Value::as_u64), Some(77)); assert_eq!(report.pointer("/summary/wrong_result").and_then(Value::as_u64), Some(0)); assert_eq!(report.pointer("/summary/incomplete").and_then(Value::as_u64), Some(0)); assert_eq!(report.pointer("/summary/blocked").and_then(Value::as_u64), Some(7)); @@ -46,11 +46,11 @@ pub(crate) fn assert_root_summary_counts(report: &Value) { ); assert_eq!( report.pointer("/summary/evidence_required_count").and_then(Value::as_u64), - Some(182) + Some(185) ); assert_eq!( report.pointer("/summary/evidence_covered_count").and_then(Value::as_u64), - Some(182) + Some(185) ); assert_eq!(report.pointer("/summary/evidence_coverage").and_then(Value::as_f64), Some(1.0)); assert_eq!(report.pointer("/summary/source_ref_coverage").and_then(Value::as_f64), Some(1.0)); @@ -68,7 +68,7 @@ pub(crate) fn assert_root_summary_counts(report: &Value) { pub(crate) fn assert_root_consolidation_summary(report: &Value) { assert_eq!( report.pointer("/summary/consolidation/proposal_count").and_then(Value::as_u64), - Some(5) + Some(6) ); assert_eq!( report.pointer("/summary/consolidation/source_mutation_count").and_then(Value::as_u64), @@ -93,3 +93,44 @@ pub(crate) fn assert_root_consolidation_summary(report: &Value) { Some(1.0) ); } + +pub(crate) fn assert_root_local_organizer_summary(report: &Value) { + assert_eq!( + report.pointer("/summary/local_organizer/job_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/summary/local_organizer/extraction_f1_not_encoded_count") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report.pointer("/summary/local_organizer/json_schema_valid_count").and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + report + .pointer("/summary/local_organizer/citation_source_ref_coverage") + .and_then(Value::as_f64), + Some(1.0) + ); + assert_eq!( + report.pointer("/summary/local_organizer/unsupported_claim_rate").and_then(Value::as_f64), + Some(0.0) + ); + assert_eq!( + report.pointer("/summary/local_organizer/source_mutation_count").and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report + .pointer("/summary/local_organizer/silent_memory_authority_mutation_count") + .and_then(Value::as_u64), + Some(0) + ); + assert_eq!( + report.pointer("/summary/local_organizer/escalation_rate").and_then(Value::as_f64), + Some(1.0) + ); +} diff --git a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_scoreboard.rs b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_scoreboard.rs index 1dbd15f7..8cd35386 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_scoreboard.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/root_aggregate_summary_scoreboard.rs @@ -80,7 +80,7 @@ fn assert_root_scoreboard_rows(report: &Value) -> Result<()> { assert_eq!(elf.pointer("/metrics/retrieval/recall_at_k").and_then(Value::as_f64), Some(0.989)); assert_eq!( elf.pointer("/metrics/retrieval/precision_at_k").and_then(Value::as_f64), - Some(0.414) + Some(0.417) ); assert_eq!(elf.pointer("/metrics/retrieval/mrr").and_then(Value::as_f64), Some(0.988)); assert_eq!(elf.pointer("/metrics/retrieval/ndcg").and_then(Value::as_f64), Some(0.985)); diff --git a/docs/evidence/benchmarking/2026-07-04-r1-local-background-organizer-benchmark-gate.md b/docs/evidence/benchmarking/2026-07-04-r1-local-background-organizer-benchmark-gate.md new file mode 100644 index 00000000..f5c0223e --- /dev/null +++ b/docs/evidence/benchmarking/2026-07-04-r1-local-background-organizer-benchmark-gate.md @@ -0,0 +1,90 @@ +--- +type: Evidence +title: "R1 Local Background Organizer Benchmark Gate - July 4, 2026" +description: "Records the fixture-backed R1 model-ladder and proposal-only local background organizer benchmark gate." +resource: docs/evidence/benchmarking/2026-07-04-r1-local-background-organizer-benchmark-gate.md +status: active +authority: evidence +owner: benchmarking +last_verified: 2026-07-04 +tags: + - docs + - evidence + - benchmarking + - model-ladder +source_refs: + - https://linear.app/hack-ink/issue/XY-1163/r1-define-model-ladder-and-local-background-organizer-benchmark-gate +code_refs: + - makefiles/benchmark-memory-b.toml + - apps/elf-eval/src/bin/real_world_job_benchmark/main.rs + - apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs + - apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json +related: + - docs/spec/agent_memory_knowledge_system_v1.md + - docs/spec/system_consolidation_proposals_v1.md +drift_watch: + - makefiles/benchmark-memory-b.toml + - apps/elf-eval/src/bin/real_world_job_benchmark/main.rs + - apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs + - apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json +--- +# R1 Local Background Organizer Benchmark Gate - July 4, 2026 + +Purpose: Record the fixture-backed R1 model-ladder and proposal-only local +background organizer benchmark gate. +Read this when: You need evidence for XY-1163 R1 model ladder and local organizer +claims. +Not this document: Provider-runtime quality proof, hosted managed-memory parity, or +Memory Authority write authorization. +Evidence command: `cargo make real-world-memory-r1-local-organizer`. + +## Result + +The R1 gate is fixture-backed and proposal-only. The canonical task runs: + +```sh +cargo make real-world-memory-r1-local-organizer +``` + +The task materializes `tmp/real-world-memory/r1-local-organizer/report.json`, validates +it with `validate-local-organizer`, and publishes +`tmp/real-world-memory/r1-local-organizer/report.md`. + +The checked-in fixture reports: + +| Metric | Value | +| --- | ---: | +| Jobs | 1 | +| Passed jobs | 1 | +| Extraction F1 | `not_encoded` with explicit blocker | +| JSON/schema valid outputs | 1 | +| Citation/source-ref coverage | 1.000 | +| Unsupported claim rate | 0.000 | +| Stale/correction/delete score | 1.000 | +| Source mutation count | 0 | +| Silent Memory Authority mutation count | 0 | +| Escalation rate | 1.000 | +| Mean latency | 42.000 ms | +| P95 latency | 42.000 ms | +| Model tier reported | L2 | +| Runtime commit provenance | `fixture-runtime@2026-07-04` | + +The resource footprint is fixture-local CPU evidence with `peak_memory_mb = 64`, +`network_required = false`, and zero USD runtime cost. + +## Claim Boundary + +This report supports only these claims: + +- ELF defines the L0/L1/L2/L3/L4 model ladder. +- ELF has a fixture-backed R1 benchmark gate for proposal-only local/background + organizer output. +- L2 local/small models may draft low-risk structured proposals only after schema, + provenance, unsupported-claim, stale/delete/correction, source-immutability, + escalation, latency, cost, and resource gates pass. + +This report does not support claims that small/local models are safe defaults for +Memory Authority writes, graph facts, current top-of-mind summaries, contradiction +repair, autonomous consolidation, benchmark judging, public claims, OpenAI Dreaming, +Letta sleep-time organization, LangMem/LangChain memory manager parity, mem0/OpenMemory +parity, Graphiti/Zep parity, or hosted managed-memory parity. diff --git a/docs/evidence/benchmarking/index.md b/docs/evidence/benchmarking/index.md index 6e17cb15..a7fcac83 100644 --- a/docs/evidence/benchmarking/index.md +++ b/docs/evidence/benchmarking/index.md @@ -62,3 +62,4 @@ Routes to: Benchmarking evidence concepts under `docs/evidence/benchmarking/`. - `2026-07-03-source-backed-quality-benchmark-harness.md`: Source-Backed Memory Quality Benchmark Harness - July 3, 2026; adds `cargo make source-backed-memory-quality` and the `elf.source_backed_memory_quality_benchmark/v1` report surface for expected evidence recall, precision@5, source-ref coverage, stale/correction/delete behavior, Context Pack activation, Recall Debug privacy, hard-fail leak counters, latency, and typed scenario coverage. - `2026-07-03-qmd-candidate-replay-comparability-gate.md`: qmd Candidate-Replay Comparability Gate - July 3, 2026; adds the Docker-aggregate `qmd-candidate-replay-comparability-gate.json` artifact and the qmd-specific typed pass/blocked gate for complete source-id mapping, held-out/leakage audit evidence, passing per-query replay rows, aggregate replay consistency, row-bound runner image digest plus product commit provenance, and no unqualified leaderboard claim. - `2026-07-03-final-source-backed-project-memory-closeout-report.md`: Final Source-Backed Project Memory Closeout Report - July 3, 2026; records XY-1157 review-readiness evidence for Source Library, Memory Authority, Source-to-Memory loop, Knowledge Workspace, Work Journal, Dreaming Review, Context Pack v1, Automatic Context Routing, Recall Engine, Recall Debug, benchmark validity, competitor claim honesty, Decodex lifecycle/status accuracy, unsupported claims, and next optimization tasks. +- `2026-07-04-r1-local-background-organizer-benchmark-gate.md`: R1 Local Background Organizer Benchmark Gate - July 4, 2026; adds `cargo make real-world-memory-r1-local-organizer`, the `elf.model_ladder/v1` and proposal-only local/background organizer benchmark gate, and explicit no-authority-write/no-parity claim boundaries for local/small model organizer output. diff --git a/docs/index.md b/docs/index.md index 16d24a14..111f22ae 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,6 +31,8 @@ The split below is by question type, not by human-versus-agent audience. - Need the source-backed project memory product boundary, P0-P5 roadmap, Decodex phase gate, Context Pack/recall boundaries, or competitor absorption rules -> `docs/spec/agent_memory_knowledge_system_v1.md` +- Need the model ladder, model profiles, or proposal-only local/background organizer + gate -> `docs/spec/system_model_ladder_background_organizer_v1.md` - Need runbooks, migrations, validation steps, troubleshooting, or operational sequences -> `docs/runbook/` - Need one-command local setup, the minimal memory+knowledge demo loop, or diff --git a/docs/log.md b/docs/log.md index 39adbef5..9cdb4339 100644 --- a/docs/log.md +++ b/docs/log.md @@ -196,3 +196,17 @@ logs. Journal, Dreaming Review, Context Pack, routing, Recall Engine, Recall Debug, benchmark validity, competitor claim boundaries, and Decodex lifecycle/status accuracy into one review-readiness evidence surface. + +## 2026-07-04 + +- Added `docs/spec/system_model_ladder_background_organizer_v1.md` for XY-1163, + defining the L0/L1/L2/L3/L4 model ladder, model profiles, local/background + organizer authority boundaries, and strong-model or explicit policy gates for + authority-changing work. +- Added `cargo make real-world-memory-r1-local-organizer`, the + `local_background_organizer` real-world fixture, benchmark report fields, and gate + validation for proposal-only local organizer output before any source or Memory + Authority mutation. +- Added the R1 local background organizer benchmark evidence report and linked the + model ladder boundary from the source-backed product, consolidation, memory + summary, and Work Journal specs. diff --git a/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md b/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md index a83c33f1..772d6af0 100644 --- a/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md +++ b/docs/runbook/benchmarking/real_world_agent_memory_benchmark.md @@ -6,7 +6,7 @@ resource: docs/runbook/benchmarking/real_world_agent_memory_benchmark.md status: active authority: procedural owner: runbook -last_verified: 2026-06-27 +last_verified: 2026-07-04 tags: - docs - runbook @@ -209,6 +209,11 @@ including the retrieval-quality slice below. The suite currently encodes: stage-level trace readback for the same-corpus gate, missing staged artifact, selected hierarchy/rejected sibling gate, and recursive expansion/pruned-branch gate so a blocker is reviewable instead of a prose-only limitation. +- `local_background_organizer`: R1 model-ladder and proposal-only local organizer + scoring for schema validity, citation/source-ref coverage, unsupported-claim rate, + stale/correction/delete behavior, source immutability, Memory Authority + immutability, escalation, latency, cost, resource footprint, and runtime + provenance. - `adversarial_quality`: stale-fact suppression, unsupported-claim refusal, conflicting source authority selection, private/excluded span suppression, and correction persistence. These fixtures gate the quality scoreboard grammar so @@ -391,8 +396,8 @@ The public quality scoreboard renders the existing manifest evidence bucket external adapter manifest is loaded, the scoreboard's typed non-pass count includes adapter coverage and scenario rows as well as fixture jobs. -Current fixture state: `cargo make real-world-memory-json` covers 81 jobs across 19 -suites, with 74 pass and 7 blocked. The adversarial quality slice contributes five +Current fixture state: `cargo make real-world-memory-json` covers 84 jobs across 20 +suites, with 77 pass and 7 blocked. The adversarial quality slice contributes five passing fixture-backed jobs that exercise stale fact suppression, unsupported-claim refusal, source-authority conflicts, private-span exclusion, and correction persistence. The P1 closeout fixture slice contributes four passing jobs for @@ -405,6 +410,10 @@ background, stale, superseded, tombstoned, and derived project-profile entries. `work_continuity` suite contributes eight passing fixture-backed Work Journal rows for reset/resume, rationale, rejected-option, explicit/inferred next-step, handoff source-ref, redaction, and janitor false-promotion boundaries. The +`local_background_organizer` suite contributes one passing R1 proposal-only model +ladder fixture with extraction F1 explicitly `not_encoded` until a labeled extraction +set exists, plus schema, provenance, stale/correction/delete, mutation, escalation, +latency, cost, resource, runtime-commit, and reproducibility metrics. The `proactive_brief` suite contributes four passing source-linked proactive suggestions and one typed private-corpus refresh blocker tied to XY-930. The blocked jobs are production-ops operator boundaries, the private-corpus refresh blocker, the @@ -415,6 +424,21 @@ readbacks plus one typed private/provider scheduler blocker tied to XY-930; it i hosted scheduler, ChatGPT Tasks, Pulse, notification, or provider-backed private-corpus parity evidence. +### R1 Local Background Organizer + +Use this task when validating the first model-ladder/background-organizer slice: + +```sh +cargo make real-world-memory-r1-local-organizer +``` + +The task writes `tmp/real-world-memory/r1-local-organizer/report.json`, validates the +local organizer gate with `validate-local-organizer`, and renders +`tmp/real-world-memory/r1-local-organizer/report.md`. This is fixture-backed +proposal-only evidence. It does not authorize local/small models to write Memory +Authority records or support Dreaming, Letta, LangMem/LangChain, mem0/OpenMemory, +Graphiti/Zep, or hosted managed-memory parity claims. + Current live-adapter state: the `elf_live_real_world` and `qmd_live_real_world` adapters run a full checked-in suite sweep through `cargo make real-world-memory-live-adapters`. Each adapter materializes generated runtime answers for 55 jobs across 13 suites before scoring, diff --git a/docs/spec/agent_memory_knowledge_system_v1.md b/docs/spec/agent_memory_knowledge_system_v1.md index 7ab7fafc..ec53f2a1 100644 --- a/docs/spec/agent_memory_knowledge_system_v1.md +++ b/docs/spec/agent_memory_knowledge_system_v1.md @@ -6,7 +6,7 @@ resource: docs/spec/agent_memory_knowledge_system_v1.md status: active authority: normative owner: spec -last_verified: 2026-07-03 +last_verified: 2026-07-04 tags: - docs - spec @@ -29,11 +29,13 @@ related: - docs/spec/system_graph_memory_postgres_v1.md - docs/spec/system_memory_summary_v1.md - docs/spec/system_work_journal_v1.md + - docs/spec/system_model_ladder_background_organizer_v1.md - docs/evidence/benchmarking/2026-07-03-source-backed-quality-benchmark-harness.md - docs/evidence/benchmarking/2026-07-03-qmd-candidate-replay-comparability-gate.md - docs/evidence/benchmarking/2026-07-03-final-source-backed-project-memory-closeout-report.md drift_watch: - docs/spec/agent_memory_knowledge_system_v1.md + - docs/spec/system_model_ladder_background_organizer_v1.md - docs/evidence/benchmarking/2026-06-23-p4-quality-hardening-productization-readiness-report.md - docs/evidence/benchmarking/2026-07-03-source-backed-quality-benchmark-harness.md - docs/evidence/benchmarking/2026-07-03-qmd-candidate-replay-comparability-gate.md @@ -213,6 +215,21 @@ implement every item in a phase at once. | P4 | Benchmark and quality hardening | Expand executable benchmarks, qmd candidate-replay comparability, adversarial jobs, public comparison grammar, quality metrics, latency/cost/resource reporting, and unsupported-claim detection. | Reports preserve job/suite/project typed states, expected evidence recall, irrelevant context ratio, unsupported claims, qmd replay comparability, and resource metrics. | | P5 | Productization | Improve local setup, agent/MCP recipes, operator UI, privacy/delete/export boundaries, and production-quality workflows. | Operator workflows have documented setup, privacy/delete/export semantics, and validation evidence without weakening source authority. | +### R1 Model Ladder Constraint + +R1 introduces the model ladder and the first local/background organizer benchmark +gate. The R1 slice is narrower than a product phase: it proves only a proposal-only +local organizer path. It must not make small/local models authoritative for Memory +Authority writes. + +R1 is governed by `docs/spec/system_model_ladder_background_organizer_v1.md` and +validated by `cargo make real-world-memory-r1-local-organizer`. A passing R1 report +may claim only that ELF defines L0/L1/L2/L3/L4 tiers and that L2 local/small models +can draft low-risk structured proposals under schema, provenance, lifecycle, +escalation, latency, cost, and source-immutability gates. Strong models or explicit +policy gates remain required for conflict resolution, multi-source synthesis, +promotion review, important corrections, benchmark judging, and public claims. + ### First Implementation Phase Constraint The first implementation issue after P0 must be the smallest coherent P1 vertical diff --git a/docs/spec/index.md b/docs/spec/index.md index cbf5e7ad..161393ab 100644 --- a/docs/spec/index.md +++ b/docs/spec/index.md @@ -47,6 +47,7 @@ Question this index answers: "what must remain true?" - `system_graph_memory_postgres_v1.md`: Graph Memory Postgres v1.0 Specification. - `system_knowledge_pages_v1.md`: Derived Knowledge Pages v1 Specification. - `system_memory_summary_v1.md`: Reviewable Memory Summary v1 Specification. +- `system_model_ladder_background_organizer_v1.md`: Model Ladder and Background Organizer v1. - `system_provenance_mapping_v1.md`: System: Note Provenance Mapping (v1). - `system_recall_debug_panel_v1.md`: Recall Debug Panel v1 Specification. - `system_search_filter_expr_v1.md`: System: Search Filter Expression Contract v1. diff --git a/docs/spec/system_consolidation_proposals_v1.md b/docs/spec/system_consolidation_proposals_v1.md index 105d2e3b..efaeac78 100644 --- a/docs/spec/system_consolidation_proposals_v1.md +++ b/docs/spec/system_consolidation_proposals_v1.md @@ -6,13 +6,14 @@ resource: docs/spec/system_consolidation_proposals_v1.md status: active authority: normative owner: spec -last_verified: 2026-06-22 +last_verified: 2026-07-04 tags: - docs - spec source_refs: [] code_refs: [] -related: [] +related: + - docs/spec/system_model_ladder_background_organizer_v1.md drift_watch: - docs/spec/system_consolidation_proposals_v1.md --- @@ -376,3 +377,10 @@ snapshots, lineage, diff, confidence, contradiction markers, and staleness marke Future derived knowledge pages may use approved proposals as input, but those pages remain rebuildable derived output. They must retain source pointers and must not become a hidden replacement for evidence-bound ELF Core memory. + +Local/background organizer proposals are governed by +`docs/spec/system_model_ladder_background_organizer_v1.md`. L2 local/small model +outputs may enter this proposal contract only as reviewable `no_op`, tag, +duplicate-merge, summary, stale-marker, or memory-candidate proposals. They must not +apply Memory Authority mutations unless a strong-model or explicit policy/reviewer gate +accepts the transition through this spec's review lifecycle. diff --git a/docs/spec/system_memory_summary_v1.md b/docs/spec/system_memory_summary_v1.md index 3cb99235..e8aec4e2 100644 --- a/docs/spec/system_memory_summary_v1.md +++ b/docs/spec/system_memory_summary_v1.md @@ -6,13 +6,14 @@ resource: docs/spec/system_memory_summary_v1.md status: active authority: normative owner: spec -last_verified: 2026-06-18 +last_verified: 2026-07-04 tags: - docs - spec source_refs: [] code_refs: [] -related: [] +related: + - docs/spec/system_model_ladder_background_organizer_v1.md drift_watch: - docs/spec/system_memory_summary_v1.md --- @@ -62,6 +63,12 @@ The artifact may include provider metadata in future lanes, but v1 summary readb does not require provider execution and must not hide source selection behind provider state. +When a local/background organizer drafts summary candidates, the model ladder in +`docs/spec/system_model_ladder_background_organizer_v1.md` applies. L2 output may +produce reviewable summary proposals only. Current top-of-mind summaries, +contradiction repair, multi-source synthesis, or public benchmark judging require an +L3 strong-model or L4 explicit policy/reviewer gate before authority-changing use. + ## Entry Categories `entries[].category` must be one of: diff --git a/docs/spec/system_model_ladder_background_organizer_v1.md b/docs/spec/system_model_ladder_background_organizer_v1.md new file mode 100644 index 00000000..c029dd57 --- /dev/null +++ b/docs/spec/system_model_ladder_background_organizer_v1.md @@ -0,0 +1,171 @@ +--- +type: Spec +title: "Model Ladder and Background Organizer v1" +description: "Define the ELF model ladder, model profiles, and proposal-only local background organizer benchmark gate." +resource: docs/spec/system_model_ladder_background_organizer_v1.md +status: active +authority: normative +owner: spec +last_verified: 2026-07-04 +tags: + - docs + - spec + - model-ladder + - background-organizer +source_refs: + - https://linear.app/hack-ink/issue/XY-1163/r1-define-model-ladder-and-local-background-organizer-benchmark-gate +code_refs: + - makefiles/benchmark-memory-b.toml + - apps/elf-eval/src/bin/real_world_job_benchmark/main.rs + - apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs + - apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json +related: + - docs/spec/agent_memory_knowledge_system_v1.md + - docs/spec/system_consolidation_proposals_v1.md + - docs/spec/system_memory_summary_v1.md + - docs/spec/system_work_journal_v1.md +drift_watch: + - docs/spec/system_model_ladder_background_organizer_v1.md + - makefiles/benchmark-memory-b.toml + - apps/elf-eval/src/bin/real_world_job_benchmark/main.rs + - apps/elf-eval/src/bin/real_world_job_benchmark/artifacts/local_organizer.rs + - apps/elf-eval/fixtures/real_world_memory/local_background_organizer/proposal_only_gate.json +--- +# Model Ladder and Background Organizer v1 + +Purpose: Define the ELF model ladder, model profiles, and proposal-only local +background organizer benchmark gate. +Status: normative +Read this when: You are implementing, validating, or reviewing model-tier use, +background organization, local extraction, memory proposal generation, or claims +about small/local models. +Not this document: Provider adapter wire formats, full Memory Authority write +semantics, or broad competitor parity claims. +Defines: `elf.model_ladder/v1`, `elf.local_background_organizer_benchmark/v1`, model +profiles, escalation policy, and R1 benchmark acceptance rules. + +## Core Rule + +Small or local models are not authoritative for Memory Authority writes. + +Local/background organizer output may draft low-risk structured proposals only. A +proposal is not current memory until an explicit review, strong-model gate, or policy +gate accepts the authority-changing action through the owning Memory Authority or +consolidation path. + +## Model Ladder + +The canonical model ladder is: + +| Tier | Name | Allowed use | Authority boundary | +| --- | --- | --- | --- | +| L0 | Deterministic code | Parsing, validation, scoring, schema checks, redaction checks, source-ref matching, and fixture materialization. | May reject or score inputs; must not infer unsupported memory facts. | +| L1 | Heuristic/lightweight local | Routing hints, duplicate candidates, simple labels, low-risk grouping, and UI triage aids. | Must emit rationale and confidence; must not create authoritative notes. | +| L2 | Schema-constrained local/small model | Proposal drafting, extraction candidates, local background organization, low-risk tags, and stale/delete/correction prompts. | Proposal-only. Must pass schema, provenance, unsupported-claim, source-mutation, and escalation gates before use. | +| L3 | Strong model | Conflict analysis, multi-source synthesis, important correction drafts, promotion-review assistance, high-impact summaries, and benchmark judging assistance. | Still review/policy gated for authority mutation; may not silently write Memory Authority. | +| L4 | Explicit policy or human gate | Final authority decisions for promotion, correction, public claims, benchmark adoption, and safety-sensitive ambiguity. | Required for authority-changing work unless a narrower accepted spec explicitly delegates the transition. | + +The tier is a capability and authority label, not a vendor claim. A model can be +configured into a lower tier when runtime evidence is weak or unavailable. + +## Model Profiles + +Model profiles are named policy slots used by benchmark and runtime configuration. +They do not require a specific provider. + +| Profile | Minimum tier | Allowed output | Required gate | +| --- | --- | --- | --- | +| `memory_extraction` | L2 | Evidence-bound candidate notes or structured proposal payloads. | JSON/schema validity, source-ref coverage, unsupported-claim rate, stale/delete/correction scoring, and L3/L4 escalation for conflicts or high-impact writes. | +| `context_routing` | L1 | Layer, scope, and budget routing hints for recall or Context Pack assembly. | Scope/lifecycle enforcement remains deterministic; routing cannot bypass read_profile or authority checks. | +| `memory_summarization` | L2 | Reviewable summaries, background brief candidates, and source-linked stale markers. | Current top-of-mind summaries, contradiction repair, and multi-source synthesis require L3/L4. | +| `benchmark_judging` | L3 | Scoring assistance, rubric critique, and claim-boundary analysis. | Final benchmark adoption and public claims require L4 policy acceptance and checked-in reproducibility evidence. | +| `promotion_review` | L3 | Review assistance for applying, correcting, or rejecting proposals. | Applying a Memory Authority mutation requires explicit review or L4 policy gate through the owning service path. | + +## Local Background Organizer + +The R1 local background organizer is read-only with respect to source and Memory +Authority storage. + +Allowed operations: + +- read bounded source, journal, note-history, proposal, and benchmark fixture inputs; +- produce schema-constrained extraction candidates and reviewable proposals; +- attach source refs, source snapshots, unsupported-claim flags, staleness markers, + contradiction markers, and escalation rationale; +- report latency, cost, and resource footprint by model tier; +- fail closed with `not_encoded`, `blocked`, or `wrong_result` evidence. + +Forbidden operations: + +- create, update, delete, deprecate, restore, or publish Memory Authority notes; +- mutate Source Library documents, Work Journal rows, source refs, search traces, + graph facts, Qdrant indexes, or benchmark evidence; +- treat a fixture-only pass as product-runtime parity with OpenAI Memory/Dreaming, + Letta sleep-time organization, LangMem/LangChain memory managers, mem0/OpenMemory, + Graphiti/Zep, or hosted managed memory. + +## Escalation Policy + +L2 output must escalate to L3 or L4 for: + +- conflict resolution; +- multi-source synthesis; +- promotion review; +- important corrections; +- benchmark judging; +- public claims; +- graph facts, current top-of-mind summaries, contradiction repair, or autonomous + consolidation. + +Escalation must be visible in proposal payloads, benchmark reports, or review traces. + +## Benchmark Gate + +The canonical R1 task is: + +```sh +cargo make real-world-memory-r1-local-organizer +``` + +The task runs fixture-backed local/background organizer jobs, validates the gate, and +publishes a Markdown report under `tmp/real-world-memory/r1-local-organizer/`. + +The generated report must include: + +- extraction F1, or an explicit `not_encoded` blocker when no labeled extraction set + exists; +- JSON/schema validity; +- citation/source-ref coverage; +- unsupported-claim rate; +- stale/correction/delete behavior; +- zero source mutation and zero silent Memory Authority mutation; +- escalation rate; +- mean and p95 latency; +- cost and resource footprint by model tier; +- product/runtime commit and reproducibility provenance when runtime adapters are + used. + +The gate fails when schema validity, citation/source-ref coverage, unsupported-claim +rate, stale/correction/delete behavior, source immutability, Memory Authority +immutability, escalation completion, latency reporting, tier resource reporting, or +runtime provenance is missing or below the fixture threshold. + +## Claim Grammar + +Allowed claims after the R1 benchmark passes: + +- ELF has a defined model ladder. +- ELF has a benchmarked proposal-only local/background organizer slice. +- Small/local models may draft low-risk structured proposals after passing the slice + gates. +- Strong models or explicit policy gates remain required for authority-changing work. + +Disallowed claims: + +- Small/local models are safe defaults for Memory Authority writes. +- Local extraction is broadly good enough for graph facts, current top-of-mind + summaries, contradiction repair, autonomous consolidation, or public benchmark + judging. +- ELF has OpenAI Dreaming, Letta sleep-time, LangMem/LangChain, mem0/OpenMemory, + Graphiti/Zep, or hosted managed-memory parity from this slice. +- Fixture-only or blocked adapter rows are product-runtime wins. diff --git a/docs/spec/system_version_registry.md b/docs/spec/system_version_registry.md index c84ea0ef..d47af87d 100644 --- a/docs/spec/system_version_registry.md +++ b/docs/spec/system_version_registry.md @@ -6,7 +6,7 @@ resource: docs/spec/system_version_registry.md status: active authority: normative owner: spec -last_verified: 2026-06-23 +last_verified: 2026-07-04 tags: - docs - spec @@ -39,6 +39,29 @@ This document is normative. When a new versioned identifier is introduced, it mu semantics, authority-layer boundaries, or claim-boundary rules become incompatible with this contract. +### Model ladder schema + +- Identifier: `elf.model_ladder/v1`. +- Type: Model-tier authority contract for deterministic, local/small, strong-model, + and explicit policy gate use. +- Defined in: `docs/spec/system_model_ladder_background_organizer_v1.md`. +- Consumers: model profile configuration, background organizer benchmarks, proposal + review policy, and agents deciding whether local/small model output is authority + changing or proposal-only. +- Bump rule: Introduce a new identifier only when tier names, allowed tier authority, + or required escalation semantics become incompatible. + +### Local background organizer benchmark schema + +- Identifier: `elf.local_background_organizer_benchmark/v1`. +- Type: R1 proposal-only local/background organizer benchmark gate. +- Defined in: `docs/spec/system_model_ladder_background_organizer_v1.md`. +- Consumers: `cargo make real-world-memory-r1-local-organizer`, + `apps/elf-eval/src/bin/real_world_job_benchmark/`, and checked-in benchmark + evidence reports. +- Bump rule: Introduce a new identifier only when required reported metrics, gate + semantics, or proposal-only claim boundaries become incompatible. + ### Agent memory quantitative scoreboard schema - Identifier: `elf.quality_scoreboard/v1`. diff --git a/docs/spec/system_work_journal_v1.md b/docs/spec/system_work_journal_v1.md index b898df9c..6037c132 100644 --- a/docs/spec/system_work_journal_v1.md +++ b/docs/spec/system_work_journal_v1.md @@ -6,7 +6,7 @@ resource: docs/spec/system_work_journal_v1.md status: active authority: normative owner: spec -last_verified: 2026-06-27 +last_verified: 2026-07-04 tags: - docs - spec @@ -23,6 +23,7 @@ related: - docs/spec/system_elf_memory_service_v2.md - docs/spec/system_consolidation_proposals_v1.md - docs/spec/system_recall_debug_panel_v1.md + - docs/spec/system_model_ladder_background_organizer_v1.md drift_watch: - packages/elf-service/src/work_journal.rs - sql/tables/042_work_journal_entries.sql @@ -50,6 +51,12 @@ Work Journal rows must not: - answer current-fact questions as authoritative memory unless the response also carries an accepted Memory Authority or Dreaming Review promotion reference. +Local/background organizer reads over Work Journal rows are governed by +`docs/spec/system_model_ladder_background_organizer_v1.md`. L2 local/small model +organizers may draft journal-to-proposal candidates with source refs, but journal rows +must not become current Memory Authority unless the candidate is accepted through the +Memory Authority or Dreaming Review promotion boundary. + ## Storage The source-of-truth table is `work_journal_entries`. diff --git a/makefiles/benchmark-memory-b.toml b/makefiles/benchmark-memory-b.toml index 283c2a1a..df242fb9 100644 --- a/makefiles/benchmark-memory-b.toml +++ b/makefiles/benchmark-memory-b.toml @@ -90,6 +90,81 @@ args = [ "tmp/source-backed-memory-quality/report.md", ] +[tasks.real-world-memory-r1-local-organizer] +workspace = false +dependencies = [ + "real-world-memory-r1-local-organizer-report", +] + +[tasks.real-world-memory-r1-local-organizer-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/local_background_organizer", + "--out", + "tmp/real-world-memory/r1-local-organizer/report.json", + "--run-id", + "real-world-memory-r1-local-organizer", + "--adapter-id", + "fixture_local_background_organizer", + "--adapter-name", + "ELF R1 local background organizer fixture", + "--adapter-behavior", + "proposal_only_schema_constrained_local_fixture", + "--adapter-storage-status", + "pass", + "--adapter-runtime-status", + "pass", + "--adapter-notes", + "Fixture-backed local/background organizer slice scores proposal-only extraction, schema, provenance, lifecycle, escalation, latency, cost, and resource metrics before any Memory Authority mutation.", +] + +[tasks.real-world-memory-r1-local-organizer-validate] +workspace = false +dependencies = [ + "real-world-memory-r1-local-organizer-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "validate-local-organizer", + "--report", + "tmp/real-world-memory/r1-local-organizer/report.json", +] + +[tasks.real-world-memory-r1-local-organizer-report] +workspace = false +dependencies = [ + "real-world-memory-r1-local-organizer-validate", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/r1-local-organizer/report.json", + "--out", + "tmp/real-world-memory/r1-local-organizer/report.md", +] + [tasks.real-world-memory-knowledge] workspace = false dependencies = [