diff --git a/agent/src/main.rs b/agent/src/main.rs index 9cca554..f48a25a 100644 --- a/agent/src/main.rs +++ b/agent/src/main.rs @@ -24,6 +24,17 @@ When given a plain-text fact or statement: 4. Use the edit_worldview tool to add or modify the appropriate line(s) 5. After editing, briefly confirm what you added +## Critical: Encode Only What Is Stated + +The Worldview file stores information the user explicitly provides for later reference. You must: + +- **Only encode information explicitly stated** in the user's fact or statement +- **Never add supplementary knowledge** from your training data (e.g., if told "gravity pulls objects toward Earth", do NOT add acceleration values, formulas, or other physics facts you know) +- **Never add assumptions or inferences** beyond what was directly stated +- **Be token-efficient** — the model already has general knowledge; we only need to store what the user specifically wants to remember + +The purpose is to capture the user's specific framing and claims, not to build a comprehensive knowledge base. General facts already exist in model weights and don't need to be stored. + Remember the design principles: state over narrative, predictability allows omission, conflict tolerance, freeform vocabulary, and LLM-native density. "#; diff --git a/evals/__init__.py b/evals/__init__.py index b7484f5..e217fe3 100644 --- a/evals/__init__.py +++ b/evals/__init__.py @@ -3,6 +3,11 @@ Tests how well various LLMs can leverage Worldview-encoded beliefs to answer questions that may challenge their base training. + +Submodules: +- common: Shared utilities (model configs, LLM clients) +- read_eval: Tests LLM response to Worldview context +- write_eval: Tests LLM document generation """ __version__ = "0.1.0" diff --git a/evals/cli.py b/evals/cli.py index 1392759..00ee187 100644 --- a/evals/cli.py +++ b/evals/cli.py @@ -21,9 +21,9 @@ def cmd_run(args): """Run evaluations.""" - from .config import ALL_MODELS, DEFAULT_MODELS, get_model, MODEL_REGISTRY - from .runner import EvalRunner, generate_report, generate_json_results - from .test_cases import ALL_TEST_CASES, Difficulty, get_cases_by_difficulty + from .common.config import ALL_MODELS, DEFAULT_MODELS, get_model, MODEL_REGISTRY + from .read_eval.runner import EvalRunner, generate_report, generate_json_results + from .read_eval.test_cases import ALL_TEST_CASES, Difficulty, get_cases_by_difficulty # Determine models to use if args.models: @@ -52,7 +52,7 @@ def cmd_run(args): print(f"Valid options: {[d.value for d in Difficulty]}") sys.exit(1) elif args.cases: - from .test_cases import get_case_by_id + from .read_eval.test_cases import get_case_by_id test_cases = [] for case_id in args.cases: tc = get_case_by_id(case_id) @@ -200,7 +200,7 @@ def cmd_write_eval(args): def cmd_list_models(args): """List available models.""" - from .config import ALL_MODELS, DEFAULT_MODELS + from .common.config import ALL_MODELS, DEFAULT_MODELS from .write_eval.runner import WRITE_MODELS, DEFAULT_WRITE_MODELS default_ids = {m.model_id for m in DEFAULT_MODELS} @@ -225,7 +225,7 @@ def cmd_list_models(args): def cmd_list_cases(args): """List available test cases.""" - from .test_cases import ALL_TEST_CASES, Difficulty + from .read_eval.test_cases import ALL_TEST_CASES, Difficulty from .write_eval.test_cases import ALL_WRITE_CASES, Complexity print("Read Evaluation Test Cases (testing LLM response to Worldview context):\n") diff --git a/evals/common/__init__.py b/evals/common/__init__.py new file mode 100644 index 0000000..be0a20b --- /dev/null +++ b/evals/common/__init__.py @@ -0,0 +1,37 @@ +""" +Common utilities shared across evaluation types. +""" + +from .config import ( + Provider, + ModelConfig, + ALL_MODELS, + DEFAULT_MODELS, + MODEL_REGISTRY, + get_model, + get_models_by_provider, + CLAUDE_SONNET, + CLAUDE_OPUS, + CLAUDE_HAIKU, + GPT_5_2, + GPT_5_MINI, +) +from .llm_clients import LLMClient, LLMResponse, create_client + +__all__ = [ + "Provider", + "ModelConfig", + "ALL_MODELS", + "DEFAULT_MODELS", + "MODEL_REGISTRY", + "get_model", + "get_models_by_provider", + "CLAUDE_SONNET", + "CLAUDE_OPUS", + "CLAUDE_HAIKU", + "GPT_5_2", + "GPT_5_MINI", + "LLMClient", + "LLMResponse", + "create_client", +] diff --git a/evals/config.py b/evals/common/config.py similarity index 100% rename from evals/config.py rename to evals/common/config.py diff --git a/evals/llm_clients.py b/evals/common/llm_clients.py similarity index 100% rename from evals/llm_clients.py rename to evals/common/llm_clients.py diff --git a/evals/read_eval/__init__.py b/evals/read_eval/__init__.py new file mode 100644 index 0000000..5364672 --- /dev/null +++ b/evals/read_eval/__init__.py @@ -0,0 +1,52 @@ +""" +Read Evaluation Module + +Tests how well LLMs can respond to Worldview-encoded beliefs. +Given worldview content as context, evaluates if the model's responses +align with the encoded beliefs. +""" + +from .evaluator import ( + EvalScore, + EvalResult, + EvalSummary, + evaluate_response, + summarize_results, +) +from .runner import ( + EvalRunner, + generate_report, + generate_json_results, +) +from .test_cases import ( + Category, + Difficulty, + ExpectedBehavior, + TestCase, + ALL_TEST_CASES, + get_cases_by_difficulty, + get_cases_by_category, + get_case_by_id, +) + +__all__ = [ + # Evaluator + "EvalScore", + "EvalResult", + "EvalSummary", + "evaluate_response", + "summarize_results", + # Runner + "EvalRunner", + "generate_report", + "generate_json_results", + # Test cases + "Category", + "Difficulty", + "ExpectedBehavior", + "TestCase", + "ALL_TEST_CASES", + "get_cases_by_difficulty", + "get_cases_by_category", + "get_case_by_id", +] diff --git a/evals/evaluator.py b/evals/read_eval/evaluator.py similarity index 97% rename from evals/evaluator.py rename to evals/read_eval/evaluator.py index 36e8784..794fb60 100644 --- a/evals/evaluator.py +++ b/evals/read_eval/evaluator.py @@ -139,6 +139,9 @@ class EvalResult: error: Optional[str] = None input_tokens: int = 0 output_tokens: int = 0 + # When CLI mode is used, stores the actual generated worldview content + # (may differ from test_case.wsl_content which is the expected/predefined content) + generated_worldview_content: Optional[str] = None @property def success(self) -> bool: diff --git a/evals/runner.py b/evals/read_eval/runner.py similarity index 91% rename from evals/runner.py rename to evals/read_eval/runner.py index 947c883..c8dbb1f 100644 --- a/evals/runner.py +++ b/evals/read_eval/runner.py @@ -13,9 +13,9 @@ from pathlib import Path from typing import Optional -from .config import ModelConfig, ALL_MODELS, DEFAULT_MODELS, get_model +from ..common.config import ModelConfig, ALL_MODELS, DEFAULT_MODELS, get_model +from ..common.llm_clients import LLMClient, LLMResponse, create_client from .evaluator import EvalResult, EvalScore, EvalSummary, evaluate_response, summarize_results -from .llm_clients import LLMClient, LLMResponse, create_client from .test_cases import ( ALL_TEST_CASES, Difficulty, @@ -136,6 +136,7 @@ def _run_single_eval( print(f" Running: {test_case.id} with {model.display_name}") # Get Worldview content + generated_content = None if self.use_cli_tool: worldview_content, error = self._generate_worldview_with_cli(test_case.fact_statement) if error: @@ -146,6 +147,7 @@ def _run_single_eval( score=EvalScore(), error=f"Worldview generation failed: {error}", ) + generated_content = worldview_content # Store for reporting else: worldview_content = test_case.wsl_content @@ -187,6 +189,7 @@ def _run_single_eval( score=score, input_tokens=response.input_tokens, output_tokens=response.output_tokens, + generated_worldview_content=generated_content, ) def run_case( @@ -340,6 +343,11 @@ def generate_report( first_result = list(model_results.values())[0] tc = first_result.test_case + # Check if any model has generated content (CLI mode was used) + has_generated_content = any( + r.generated_worldview_content for r in model_results.values() + ) + lines.extend([ f"### {tc.name}", "", @@ -349,12 +357,32 @@ def generate_report( "", f"> {tc.fact_statement}", "", - "#### 2. Worldview Content", - "", - "```wvf", - tc.wsl_content.strip(), - "```", - "", + ]) + + # Show worldview content (CLI-generated if available, otherwise predefined) + lines.append("#### 2. Worldview Content") + lines.append("") + if has_generated_content: + # Show CLI-generated content + for r in model_results.values(): + if r.generated_worldview_content: + lines.extend([ + "```wvf", + r.generated_worldview_content.strip(), + "```", + "", + ]) + break + else: + # Show predefined content (non-CLI mode) + lines.extend([ + "```wvf", + tc.wsl_content.strip(), + "```", + "", + ]) + + lines.extend([ "#### 3. Question Asked", "", f"> **{tc.question}**", @@ -465,6 +493,7 @@ def generate_json_results( "aligned": r.score.aligned_with_worldview, }, "response_preview": r.response[:200] if r.response else None, + "generated_worldview_content": r.generated_worldview_content, } for r in results ], diff --git a/evals/test_cases.py b/evals/read_eval/test_cases.py similarity index 100% rename from evals/test_cases.py rename to evals/read_eval/test_cases.py diff --git a/evals/worldview_prompt.py b/evals/read_eval/worldview_prompt.py similarity index 100% rename from evals/worldview_prompt.py rename to evals/read_eval/worldview_prompt.py diff --git a/evals/write_eval/__init__.py b/evals/write_eval/__init__.py index a39eb9f..d8996c7 100644 --- a/evals/write_eval/__init__.py +++ b/evals/write_eval/__init__.py @@ -5,4 +5,55 @@ and update Worldview documents using the agent CLI. """ -__version__ = "0.1.0" +from .evaluator import ( + AgentMetrics, + WriteScore, + WriteResult, + WriteSummary, + evaluate_write, + summarize_write_results, +) +from .runner import ( + WriteEvalRunner, + WRITE_MODELS, + DEFAULT_WRITE_MODELS, + get_write_model, + generate_write_report, + generate_write_json, +) +from .test_cases import ( + Complexity, + TaskType, + ExpectedStructure, + WriteTestCase, + ALL_WRITE_CASES, + get_cases_by_complexity, + get_cases_by_task_type, + get_case_by_id, +) + +__all__ = [ + # Evaluator + "AgentMetrics", + "WriteScore", + "WriteResult", + "WriteSummary", + "evaluate_write", + "summarize_write_results", + # Runner + "WriteEvalRunner", + "WRITE_MODELS", + "DEFAULT_WRITE_MODELS", + "get_write_model", + "generate_write_report", + "generate_write_json", + # Test cases + "Complexity", + "TaskType", + "ExpectedStructure", + "WriteTestCase", + "ALL_WRITE_CASES", + "get_cases_by_complexity", + "get_cases_by_task_type", + "get_case_by_id", +]