Show expected vs actual worldview content in read eval reports

agent/src/main.rs

@@ -24,6 +24,17 @@ When given a plain-text fact or statement:
 4. Use the edit_worldview tool to add or modify the appropriate line(s)
 5. After editing, briefly confirm what you added
 
+## Critical: Encode Only What Is Stated
+
+The Worldview file stores information the user explicitly provides for later reference. You must:
+
+- **Only encode information explicitly stated** in the user's fact or statement
+- **Never add supplementary knowledge** from your training data (e.g., if told "gravity pulls objects toward Earth", do NOT add acceleration values, formulas, or other physics facts you know)
+- **Never add assumptions or inferences** beyond what was directly stated
+- **Be token-efficient** — the model already has general knowledge; we only need to store what the user specifically wants to remember
+
+The purpose is to capture the user's specific framing and claims, not to build a comprehensive knowledge base. General facts already exist in model weights and don't need to be stored.
+
 Remember the design principles: state over narrative, predictability allows omission, conflict tolerance, freeform vocabulary, and LLM-native density.
 "#;
 

evals/__init__.py

@@ -3,6 +3,11 @@
 
 Tests how well various LLMs can leverage Worldview-encoded beliefs
 to answer questions that may challenge their base training.
+
+Submodules:
+- common: Shared utilities (model configs, LLM clients)
+- read_eval: Tests LLM response to Worldview context
+- write_eval: Tests LLM document generation
 """
 
 __version__ = "0.1.0"

evals/cli.py

@@ -21,9 +21,9 @@
 
 def cmd_run(args):
     """Run evaluations."""
-    from .config import ALL_MODELS, DEFAULT_MODELS, get_model, MODEL_REGISTRY
-    from .runner import EvalRunner, generate_report, generate_json_results
-    from .test_cases import ALL_TEST_CASES, Difficulty, get_cases_by_difficulty
+    from .common.config import ALL_MODELS, DEFAULT_MODELS, get_model, MODEL_REGISTRY
+    from .read_eval.runner import EvalRunner, generate_report, generate_json_results
+    from .read_eval.test_cases import ALL_TEST_CASES, Difficulty, get_cases_by_difficulty
 
     # Determine models to use
     if args.models:
@@ -52,7 +52,7 @@ def cmd_run(args):
             print(f"Valid options: {[d.value for d in Difficulty]}")
             sys.exit(1)
     elif args.cases:
-        from .test_cases import get_case_by_id
+        from .read_eval.test_cases import get_case_by_id
         test_cases = []
         for case_id in args.cases:
             tc = get_case_by_id(case_id)
@@ -200,7 +200,7 @@ def cmd_write_eval(args):
 
 def cmd_list_models(args):
     """List available models."""
-    from .config import ALL_MODELS, DEFAULT_MODELS
+    from .common.config import ALL_MODELS, DEFAULT_MODELS
     from .write_eval.runner import WRITE_MODELS, DEFAULT_WRITE_MODELS
 
     default_ids = {m.model_id for m in DEFAULT_MODELS}
@@ -225,7 +225,7 @@ def cmd_list_models(args):
 
 def cmd_list_cases(args):
     """List available test cases."""
-    from .test_cases import ALL_TEST_CASES, Difficulty
+    from .read_eval.test_cases import ALL_TEST_CASES, Difficulty
     from .write_eval.test_cases import ALL_WRITE_CASES, Complexity
 
     print("Read Evaluation Test Cases (testing LLM response to Worldview context):\n")

evals/common/__init__.py

@@ -0,0 +1,37 @@
+"""
+Common utilities shared across evaluation types.
+"""
+
+from .config import (
+    Provider,
+    ModelConfig,
+    ALL_MODELS,
+    DEFAULT_MODELS,
+    MODEL_REGISTRY,
+    get_model,
+    get_models_by_provider,
+    CLAUDE_SONNET,
+    CLAUDE_OPUS,
+    CLAUDE_HAIKU,
+    GPT_5_2,
+    GPT_5_MINI,
+)
+from .llm_clients import LLMClient, LLMResponse, create_client
+
+__all__ = [
+    "Provider",
+    "ModelConfig",
+    "ALL_MODELS",
+    "DEFAULT_MODELS",
+    "MODEL_REGISTRY",
+    "get_model",
+    "get_models_by_provider",
+    "CLAUDE_SONNET",
+    "CLAUDE_OPUS",
+    "CLAUDE_HAIKU",
+    "GPT_5_2",
+    "GPT_5_MINI",
+    "LLMClient",
+    "LLMResponse",
+    "create_client",
+]

evals/read_eval/__init__.py

@@ -0,0 +1,52 @@
+"""
+Read Evaluation Module
+
+Tests how well LLMs can respond to Worldview-encoded beliefs.
+Given worldview content as context, evaluates if the model's responses
+align with the encoded beliefs.
+"""
+
+from .evaluator import (
+    EvalScore,
+    EvalResult,
+    EvalSummary,
+    evaluate_response,
+    summarize_results,
+)
+from .runner import (
+    EvalRunner,
+    generate_report,
+    generate_json_results,
+)
+from .test_cases import (
+    Category,
+    Difficulty,
+    ExpectedBehavior,
+    TestCase,
+    ALL_TEST_CASES,
+    get_cases_by_difficulty,
+    get_cases_by_category,
+    get_case_by_id,
+)
+
+__all__ = [
+    # Evaluator
+    "EvalScore",
+    "EvalResult",
+    "EvalSummary",
+    "evaluate_response",
+    "summarize_results",
+    # Runner
+    "EvalRunner",
+    "generate_report",
+    "generate_json_results",
+    # Test cases
+    "Category",
+    "Difficulty",
+    "ExpectedBehavior",
+    "TestCase",
+    "ALL_TEST_CASES",
+    "get_cases_by_difficulty",
+    "get_cases_by_category",
+    "get_case_by_id",
+]

evals/evaluator.py → evals/read_eval/evaluator.py

@@ -139,6 +139,9 @@ class EvalResult:
     error: Optional[str] = None
     input_tokens: int = 0
     output_tokens: int = 0
+    # When CLI mode is used, stores the actual generated worldview content
+    # (may differ from test_case.wsl_content which is the expected/predefined content)
+    generated_worldview_content: Optional[str] = None
 
     @property
     def success(self) -> bool:

evals/runner.py → evals/read_eval/runner.py

@@ -13,9 +13,9 @@
 from pathlib import Path
 from typing import Optional
 
-from .config import ModelConfig, ALL_MODELS, DEFAULT_MODELS, get_model
+from ..common.config import ModelConfig, ALL_MODELS, DEFAULT_MODELS, get_model
+from ..common.llm_clients import LLMClient, LLMResponse, create_client
 from .evaluator import EvalResult, EvalScore, EvalSummary, evaluate_response, summarize_results
-from .llm_clients import LLMClient, LLMResponse, create_client
 from .test_cases import (
     ALL_TEST_CASES,
     Difficulty,
@@ -136,6 +136,7 @@ def _run_single_eval(
             print(f"  Running: {test_case.id} with {model.display_name}")
 
         # Get Worldview content
+        generated_content = None
         if self.use_cli_tool:
             worldview_content, error = self._generate_worldview_with_cli(test_case.fact_statement)
             if error:
@@ -146,6 +147,7 @@ def _run_single_eval(
                     score=EvalScore(),
                     error=f"Worldview generation failed: {error}",
                 )
+            generated_content = worldview_content  # Store for reporting
         else:
             worldview_content = test_case.wsl_content
 
@@ -187,6 +189,7 @@ def _run_single_eval(
             score=score,
             input_tokens=response.input_tokens,
             output_tokens=response.output_tokens,
+            generated_worldview_content=generated_content,
         )
 
     def run_case(
@@ -340,6 +343,11 @@ def generate_report(
         first_result = list(model_results.values())[0]
         tc = first_result.test_case
 
+        # Check if any model has generated content (CLI mode was used)
+        has_generated_content = any(
+            r.generated_worldview_content for r in model_results.values()
+        )
+
         lines.extend([
             f"### {tc.name}",
             "",
@@ -349,12 +357,32 @@ def generate_report(
             "",
             f"> {tc.fact_statement}",
             "",
-            "#### 2. Worldview Content",
-            "",
-            "```wvf",
-            tc.wsl_content.strip(),
-            "```",
-            "",
+        ])
+
+        # Show worldview content (CLI-generated if available, otherwise predefined)
+        lines.append("#### 2. Worldview Content")
+        lines.append("")
+        if has_generated_content:
+            # Show CLI-generated content
+            for r in model_results.values():
+                if r.generated_worldview_content:
+                    lines.extend([
+                        "```wvf",
+                        r.generated_worldview_content.strip(),
+                        "```",
+                        "",
+                    ])
+                    break
+        else:
+            # Show predefined content (non-CLI mode)
+            lines.extend([
+                "```wvf",
+                tc.wsl_content.strip(),
+                "```",
+                "",
+            ])
+
+        lines.extend([
             "#### 3. Question Asked",
             "",
             f"> **{tc.question}**",
@@ -465,6 +493,7 @@ def generate_json_results(
                         "aligned": r.score.aligned_with_worldview,
                     },
                     "response_preview": r.response[:200] if r.response else None,
+                    "generated_worldview_content": r.generated_worldview_content,
                 }
                 for r in results
             ],

evals/write_eval/__init__.py

@@ -5,4 +5,55 @@
 and update Worldview documents using the agent CLI.
 """
 
-__version__ = "0.1.0"
+from .evaluator import (
+    AgentMetrics,
+    WriteScore,
+    WriteResult,
+    WriteSummary,
+    evaluate_write,
+    summarize_write_results,
+)
+from .runner import (
+    WriteEvalRunner,
+    WRITE_MODELS,
+    DEFAULT_WRITE_MODELS,
+    get_write_model,
+    generate_write_report,
+    generate_write_json,
+)
+from .test_cases import (
+    Complexity,
+    TaskType,
+    ExpectedStructure,
+    WriteTestCase,
+    ALL_WRITE_CASES,
+    get_cases_by_complexity,
+    get_cases_by_task_type,
+    get_case_by_id,
+)
+
+__all__ = [
+    # Evaluator
+    "AgentMetrics",
+    "WriteScore",
+    "WriteResult",
+    "WriteSummary",
+    "evaluate_write",
+    "summarize_write_results",
+    # Runner
+    "WriteEvalRunner",
+    "WRITE_MODELS",
+    "DEFAULT_WRITE_MODELS",
+    "get_write_model",
+    "generate_write_report",
+    "generate_write_json",
+    # Test cases
+    "Complexity",
+    "TaskType",
+    "ExpectedStructure",
+    "WriteTestCase",
+    "ALL_WRITE_CASES",
+    "get_cases_by_complexity",
+    "get_cases_by_task_type",
+    "get_case_by_id",
+]