diff --git a/contracts/model/decision-questions.json b/contracts/model/decision-questions.json index 817d7d7..1eac821 100644 --- a/contracts/model/decision-questions.json +++ b/contracts/model/decision-questions.json @@ -17,6 +17,11 @@ "train": 382, "validation": 103 }, + "closure.outcome": { + "test": 115, + "train": 382, + "validation": 103 + }, "closure.withdrawn": { "test": 115, "train": 382, @@ -26,24 +31,29 @@ "model": "typesafe/jev-1.13", "test": { "closure.deadline_changed": { - "accuracy": 0.9565217391304348, - "brier": 0.04520347826086957, - "ece_10": 0.08695652173913043 + "accuracy": 0.9304347826086956, + "brier": 0.06380869565217391, + "ece_10": 0.12939130434782606 }, "closure.fulfilled": { "accuracy": 0.8695652173913043, - "brier": 0.1078886956521739, - "ece_10": 0.15686956521739134 + "brier": 0.09252521739130433, + "ece_10": 0.13947826086956522 }, "closure.modified": { - "accuracy": 0.4782608695652174, - "brier": 0.33723652173913055, - "ece_10": 0.35113043478260875 + "accuracy": 0.5043478260869565, + "brier": 0.33256173913043485, + "ece_10": 0.31530434782608696 + }, + "closure.outcome": { + "accuracy": 0.8260869565217391, + "brier": 0.2776495652173913, + "ece_10": 0.10530434782608707 }, "closure.withdrawn": { "accuracy": 1.0, - "brier": 0.01063304347826087, - "ece_10": 0.08069565217391303 + "brier": 0.014983478260869564, + "ece_10": 0.09573913043478262 } } }, @@ -83,34 +93,34 @@ "model": "typesafe/jev-1.13", "test": { "rules.deadline_kind": { - "accuracy": 0.7349397590361446, - "brier": 0.42919518072289153, - "ece_10": 0.19192771084337357 + "accuracy": 0.7590361445783133, + "brier": 0.3796602409638554, + "ece_10": 0.15795180722891577 }, "rules.duplicate_action": { - "accuracy": 0.9012345679012346, - "brier": 0.08698271604938271, - "ece_10": 0.11061728395061728 + "accuracy": 0.9135802469135802, + "brier": 0.08201111111111112, + "ece_10": 0.09345679012345678 }, "rules.event_match": { - "accuracy": 0.8235294117647058, - "brier": 0.13567176470588233, - "ece_10": 0.11494117647058823 + "accuracy": 0.788235294117647, + "brier": 0.16230352941176474, + "ece_10": 0.16776470588235287 }, "rules.recap": { "accuracy": 1.0, - "brier": 0.014240740740740738, - "ece_10": 0.09320987654320988 + "brier": 0.0151, + "ece_10": 0.09444444444444446 }, "rules.scoped_event": { "accuracy": 0.8928571428571429, - "brier": 0.08740357142857143, - "ece_10": 0.11821428571428574 + "brier": 0.08850833333333333, + "ece_10": 0.11845238095238095 }, "rules.thread_merge": { "accuracy": 0.9054054054054054, - "brier": 0.08270810810810811, - "ece_10": 0.10864864864864865 + "brier": 0.05593783783783782, + "ece_10": 0.08297297297297303 } } }, @@ -150,34 +160,34 @@ "model": "typesafe/jev-1.13", "test": { "triage.asks_question": { - "accuracy": 0.8051948051948052, - "brier": 0.1256051948051948, - "ece_10": 0.17636363636363633 + "accuracy": 0.7532467532467533, + "brier": 0.15356883116883116, + "ece_10": 0.19662337662337664 }, "triage.asks_recipient": { - "accuracy": 0.9102564102564102, - "brier": 0.05079358974358975, - "ece_10": 0.10653846153846154 + "accuracy": 0.9487179487179487, + "brier": 0.037028205128205136, + "ece_10": 0.09461538461538474 }, "triage.automated_notification": { "accuracy": 0.9887640449438202, - "brier": 0.016422471910112358, - "ece_10": 0.04786516853932592 + "brier": 0.014239325842696628, + "ece_10": 0.03831460674157311 }, "triage.boilerplate": { "accuracy": 0.9864864864864865, - "brier": 0.006682432432432433, - "ece_10": 0.05337837837837836 + "brier": 0.009335135135135135, + "ece_10": 0.05837837837837828 }, "triage.commits_sender": { - "accuracy": 0.8625, - "brier": 0.09970500000000002, - "ece_10": 0.15899999999999995 + "accuracy": 0.85, + "brier": 0.11351625000000001, + "ece_10": 0.18037500000000004 }, "triage.names_time": { - "accuracy": 0.8472222222222222, - "brier": 0.12589166666666665, - "ece_10": 0.18000000000000002 + "accuracy": 0.9027777777777778, + "brier": 0.07234722222222222, + "ece_10": 0.09555555555555552 } } } @@ -202,39 +212,39 @@ "type": "choice" }, "closure.deadline_changed": { - "accept": 0.7, - "escalate": 0.65, + "accept": 0.95, + "escalate": 0.5, "instructions": "The later text sets a different deadline for the obligation.", "type": "noul" }, "closure.fulfilled": { - "accept": 0.7, - "escalate": 0.65, + "accept": 0.93, + "escalate": 0.5, "instructions": "The later text shows the obligation has been carried out.", "type": "noul" }, "closure.modified": { - "accept": 0.7, + "accept": 0.98, "escalate": 0.2, "instructions": "The later text changes what the obligation requires.", "type": "noul" }, "closure.outcome": { - "accept": 0.7, - "escalate": 0.3, + "accept": 1.0, + "escalate": 0.8, "instructions": "Which single outcome, if any, does the later paragraph express for the obligation?", "options": { - "fulfilled": "The later paragraph says the obligation was carried out.", - "withdrawn": "The later paragraph cancels or withdraws the obligation.", "deadline_changed": "The later paragraph sets a different deadline for the obligation.", + "fulfilled": "The later paragraph says the obligation was carried out.", "modified": "The later paragraph changes what the obligation requires.", - "none": "The later paragraph expresses none of the listed outcomes for the obligation." + "none": "The later paragraph expresses none of the listed outcomes for the obligation.", + "withdrawn": "The later paragraph cancels or withdraws the obligation." }, "type": "choice" }, "closure.withdrawn": { "accept": 0.5, - "escalate": 0.45, + "escalate": 0.33, "instructions": "The later text cancels or withdraws the obligation.", "type": "noul" }, @@ -243,12 +253,12 @@ "escalate": 0.3, "instructions": "Which single claim type, if any, does the paragraph express?", "options": { - "request": "The paragraph asks its recipient to do something.", - "promise": "The paragraph commits its sender to doing something.", - "question": "The paragraph asks a genuine question that seeks an answer.", "attribution": "The paragraph attributes an obligation or commitment to another party.", "delegation": "The paragraph delegates an obligation from one party to another.", - "none": "The paragraph expresses none of the listed claim types." + "none": "The paragraph expresses none of the listed claim types.", + "promise": "The paragraph commits its sender to doing something.", + "question": "The paragraph asks a genuine question that seeks an answer.", + "request": "The paragraph asks its recipient to do something." }, "type": "choice" }, @@ -271,9 +281,9 @@ "type": "choice" }, "rules.deadline_kind": { - "accept": 0.7, + "accept": 1.0, "escalate": 0.65, - "instructions": "Classify how the phrase expresses a deadline.", + "instructions": "The phrase is classified as event-tied, soft, or unknown regarding its deadline timing.", "options": { "event_tied": "The deadline is tied to a named event.", "soft": "The timing is flexible or aspirational.", @@ -282,72 +292,72 @@ "type": "choice" }, "rules.duplicate_action": { - "accept": 0.7, - "escalate": 0.65, - "instructions": "The action sentence `action_a` asks for the same thing as the action sentence `action_b`.", + "accept": 0.9, + "escalate": 0.36, + "instructions": "action_a and action_b are semantically equivalent.", "type": "noul" }, "rules.event_match": { - "accept": 0.7, - "escalate": 0.6, - "instructions": "The phrase refers to the event name.", + "accept": 0.96, + "escalate": 0.5, + "instructions": "The phrase refers to the event_name.", "type": "noul" }, "rules.recap": { "accept": 0.5, - "escalate": 0.45, + "escalate": 0.4, "instructions": "This email is an automatically generated meeting summary, recap, or transcript.", "type": "noul" }, "rules.scoped_event": { - "accept": 0.85, - "escalate": 0.75, + "accept": 0.91, + "escalate": 0.395, "instructions": "This request is about attending, preparing for, or bringing something to a meeting or event.", "type": "noul" }, "rules.thread_merge": { - "accept": 0.85, - "escalate": 0.8, - "instructions": "The topic of first_paragraph_a is the topic of first_paragraph_b.", + "accept": 0.915, + "escalate": 0.5, + "instructions": "The conversation topic of first_paragraph_a is identical to that of first_paragraph_b.", "type": "noul" }, "triage.asks_question": { "accept": 0.95, - "escalate": 0.9, - "instructions": "The paragraph_text asks a genuine question that seeks an answer.", + "escalate": 0.88, + "instructions": "The paragraph_text asks a genuine question seeking an answer.", "type": "noul" }, "triage.asks_recipient": { - "accept": 0.5, - "escalate": 0.45, - "instructions": "The paragraph_text explicitly instructs its recipient to perform an action.", + "accept": 0.9, + "escalate": 0.8, + "instructions": "The paragraph_text directly requests the recipient to perform an action.", "type": "noul" }, "triage.automated_notification": { "accept": 0.5, - "escalate": 0.45, + "escalate": 0.4, "instructions": "The paragraph is an automatically generated notification.", "type": "noul" }, "triage.boilerplate": { "accept": 0.5, - "escalate": 0.45, + "escalate": 0.4, "instructions": "The paragraph is a signature, legal footer, unsubscribe notice, or disclaimer.", "type": "noul" }, "triage.commits_sender": { - "accept": 0.9, - "escalate": 0.85, - "instructions": "The paragraph commits its sender to doing something.", + "accept": 0.93, + "escalate": 0.5, + "instructions": "The paragraph_text commits the user to an action.", "type": "noul" }, "triage.names_time": { - "accept": 0.7, - "escalate": 0.65, - "instructions": "The paragraph names a time, date, deadline, or event-relative time.", + "accept": 0.955, + "escalate": 0.5, + "instructions": "The paragraph_text names a time, date, or deadline.", "type": "noul" } }, "schema_version": 1, - "tuned_at": "2026-09-20T21:50:51.564400+00:00" + "tuned_at": "2026-09-27T15:18:48.313117+00:00" } diff --git a/tools/check-model-boundary.ps1 b/tools/check-model-boundary.ps1 index 0f62a45..ace9947 100644 --- a/tools/check-model-boundary.ps1 +++ b/tools/check-model-boundary.ps1 @@ -287,7 +287,7 @@ $decisionQuestions = Read-Json $DecisionQuestionsPath 'P0-MODEL-DECISIONS-001' if ($null -ne $decisionQuestions) { $decisionCanonical = $decisionQuestions | ConvertTo-Json -Depth 100 -Compress $decisionHash = Sha256-Hex $decisionCanonical - if ($decisionHash -ne '4bd00f0347538f3e91865079e36f70bee8e814685e92382840e22cea6c763ce2' -or $decisionQuestions.schema_version -ne 1 -or $decisionQuestions.model -ne 'typesafe/jev-1.13') { Fail 'P0-MODEL-DECISIONS-001' } + if ($decisionHash -ne '2a4323b958ba671f35791be4a8392e6c831b368c0cae23420eaf9a3c1c122539' -or $decisionQuestions.schema_version -ne 1 -or $decisionQuestions.model -ne 'typesafe/jev-1.13') { Fail 'P0-MODEL-DECISIONS-001' } $questionsProperty = $decisionQuestions.PSObject.Properties['questions'] if ($null -eq $questionsProperty -or @($questionsProperty.Value.PSObject.Properties).Count -eq 0) { Fail 'P0-MODEL-DECISIONS-001' diff --git a/tools/jev-optimize/README.md b/tools/jev-optimize/README.md index afcb2e2..50e7559 100644 --- a/tools/jev-optimize/README.md +++ b/tools/jev-optimize/README.md @@ -24,6 +24,12 @@ the model that authors synthetic rows. The Decisions endpoint receives `provider: {"zdr": true}` unless a content-free probe shows that the alpha endpoint rejects that member. The reflection endpoint always requests ZDR. +`JevLM` sends the production wire shape: flat state fields, the registry question +ID as the question key, and the DSPy signature docstring in that question's +`instructions`. `tests/test_offline_end_to_end.py` runs the real GEPA and +ReAnchor path against a fake Decisions endpoint and is the required offline gate +before any paid optimization run. + ## Commands ```powershell @@ -87,9 +93,12 @@ possible among `event_tied`, `soft`, and `unknown` (34/33/33 at 100 rows). For every applicable binary label, `from_user` differs by at most 0.1 between positive and negative rows and the `days_later` means differ by at most 2 days. Each corpus has at least 300 distinct paragraph texts, and closure has at least -40 distinct obligation texts. The threshold tuner requires at least 20 positives -and 20 negatives in its sweep; -smaller samples retain the registry defaults. +40 distinct obligation texts. Calibration requires at least 20 positives and +20 negatives; smaller samples retain the registry defaults. For Noul questions, +DSPy's ReAnchor runs twice on train plus validation after GEPA: one metric heavily +penalizes false positives to fit the accept threshold, and the mirrored metric +heavily penalizes false negatives to fit the escalation threshold. Choice +questions continue to use the confidence-based selective-classification sweep. LLM generation uses a fixed matrix of at least 30 scenario seeds per question, sends a separate prompt and strict schema for each question batch, assigns @@ -138,9 +147,16 @@ already outside this repository by construction (the exporter refuses to write inside it), so no extra step is needed to keep them out of `git`. The registry hash printed by `write-registry` is SHA-256 over -`json.dumps(obj, separators=(",", ":"), ensure_ascii=False)`. The registry is -written with sorted keys, so PowerShell's `ConvertTo-Json -Depth 100 -Compress` -preserves the same member order. On the hand-written bootstrap file the hashes -do differ: PowerShell preserves the decimal scale (`0.70`) while Python emits -`0.7`. After `write-registry` normalizes the file through Python, the two -recipes serialize parsed numeric values identically. +`json.dumps(obj, separators=(",", ":"), ensure_ascii=False)`. It is not the +value to pin in `tools/check-model-boundary.ps1`. That checker hashes +`ConvertTo-Json -Depth 100 -Compress` of the parsed file, and PowerShell parses +`tuned_at` into a DateTime and re-emits it in the machine's local offset, so the +two recipes differ in that one member (and the checker's value depends on the +timezone of the machine that computes it). Pin the checker's own value: + +```powershell +$c = Get-Content -Raw contracts/model/decision-questions.json | ConvertFrom-Json +$json = $c | ConvertTo-Json -Depth 100 -Compress +$sha = [System.Security.Cryptography.SHA256]::Create() +($sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($json)) | ForEach-Object { $_.ToString('x2') }) -join '' +``` diff --git a/tools/jev-optimize/jev_optimize/adapter.py b/tools/jev-optimize/jev_optimize/adapter.py index e0316bc..027302a 100644 --- a/tools/jev-optimize/jev_optimize/adapter.py +++ b/tools/jev-optimize/jev_optimize/adapter.py @@ -1,4 +1,4 @@ -"""DSPy adapter that executes one typed Jev request per signature call.""" +"""Thin DSPy system-one LM for OpenRouter Decisions.""" from __future__ import annotations @@ -6,77 +6,54 @@ import dspy -from .questions import SPECS, jev_question +from .questions import SPECS -class JevLM(dspy.BaseLM): - """Marker LM; typed execution is handled by :class:`JevAdapter`.""" +class JevLM: + """Expose the Decisions transport through DSPy's decision-request protocol.""" - def __init__(self) -> None: - super().__init__(model="typesafe/jev", model_type="chat", cache=False) - - def forward(self, *_args: Any, **_kwargs: Any) -> Any: - raise RuntimeError("JevLM requires JevAdapter") - - -class JevAdapter(dspy.Adapter): - """Translate a DSPy signature directly to Jev state and typed questions.""" + supports_decision_requests = True + cache = True def __init__(self, client: Any, question_id: str) -> None: - super().__init__() self.client = client self.question_id = question_id - def __deepcopy__(self, memo: dict[int, Any]) -> JevAdapter: - """Copy adapter metadata while deliberately sharing the transport client.""" + def __deepcopy__(self, memo: dict[int, Any]) -> JevLM: + """Copy LM metadata while deliberately sharing the transport client.""" copied = type(self)(self.client, self.question_id) memo[id(self)] = copied return copied def __call__( - self, - lm: Any, - lm_kwargs: dict[str, Any], - signature: type[dspy.Signature], - demos: list[dict[str, Any]], - inputs: dict[str, Any], - ) -> list[dict[str, Any]]: - del lm, lm_kwargs, demos - question = jev_question(self.question_id) - question["instructions"] = signature.instructions - answer = self.client.decide(inputs, {self.question_id: question})[self.question_id] - if answer["type"] == "noul": - return [{"probability": float(answer["noul"])}] - return [ - { - "choice": answer["choice"], - "probabilities": answer["probabilities"], - "confidence": answer["confidence"], + self, state: dict[str, Any], questions: dict[str, Any] + ) -> dict[str, Any]: + if set(questions) != {"decision"}: + raise ValueError("JevLM requires exactly one 'decision' question") + flat_state = state["inputs"] + production_questions = { + self.question_id: { + **questions["decision"], + "instructions": state["instructions"], } - ] - - -class JevPredict(dspy.Module): - """A GEPA-optimizable predictor backed by Jev rather than a chat model.""" - - def __init__(self, question_id: str, client: Any) -> None: - super().__init__() - self.predict = dspy.Predict(SPECS[question_id].signature) - self.adapter = JevAdapter(client, question_id) - self.lm = JevLM() - - def forward(self, **kwargs: Any) -> dspy.Prediction: - inputs = {name: kwargs[name] for name in self.predict.signature.input_fields} - with dspy.context(adapter=self.adapter, lm=self.lm): - return self.predict(**inputs) - - -class JevPrograms: - """Factory for per-question GEPA programs sharing one Decisions client.""" - - def __init__(self, client: Any) -> None: - self.client = client - - def program(self, question_id: str) -> JevPredict: - return JevPredict(question_id, self.client) + } + request = { + "provider": "jev", + "state": flat_state, + "questions": production_questions, + } + response = dspy.cache.get(request) if self.cache else None + if response is None: + response = self.client.decide(flat_state, production_questions) + if self.cache: + dspy.cache.put(request, response) + return {"decision": response[self.question_id]} + + +def program(question_id: str, client: Any) -> dspy.Predict: + """Build one native decision predictor and bind it to the shared transport.""" + + predictor = dspy.Predict(SPECS[question_id].signature) + predictor.set_lm(JevLM(client, question_id)) + return predictor diff --git a/tools/jev-optimize/jev_optimize/cli.py b/tools/jev-optimize/jev_optimize/cli.py index c80e788..36a5aed 100644 --- a/tools/jev-optimize/jev_optimize/cli.py +++ b/tools/jev-optimize/jev_optimize/cli.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any -from .adapter import JevPrograms +from .adapter import program from .client import DecisionsClient, ReplayClient from .data import fetch_enron, load_jsonl, split_rows from .optimize import evaluate_question, optimize_set @@ -101,11 +101,11 @@ def main() -> None: print(f"zdr_member={str(DecisionsClient().probe()).lower()}") elif args.command == "evaluate": rows = load_jsonl(args.data) - adapter = JevPrograms(_client(args.replay)) + client = _client(args.replay) registry = json.loads(DEFAULT_REGISTRY.read_text(encoding="utf-8")) result = { question_id: evaluate_question( - adapter.program(question_id), + program(question_id, client), question_id, split_rows(rows, question_id=question_id)["test"], registry_thresholds={ diff --git a/tools/jev-optimize/jev_optimize/metrics.py b/tools/jev-optimize/jev_optimize/metrics.py index 7c0e2ae..0af4b40 100644 --- a/tools/jev-optimize/jev_optimize/metrics.py +++ b/tools/jev-optimize/jev_optimize/metrics.py @@ -73,8 +73,8 @@ def threshold_sweep(labels: list[bool], probabilities: list[float]) -> list[dict return rows -# A noul below 0.5 leans false; accepting it as true would be wrong whatever -# the sweep says. Escalation covers the band between the floors. +# Choice confidence below 0.5 is not accepted even if a small calibration +# sample looks clean. Escalation covers the band between the floors. ACCEPT_FLOOR = 0.5 ESCALATE_FLOOR = 0.2 @@ -88,8 +88,9 @@ def choose_thresholds( defaults (0.70, 0.30) are returned with ``insufficient_data``. The accept threshold is the smallest threshold of at least ``ACCEPT_FLOOR`` whose selective risk is within the limit and whose positive recall is at least 0.5: a probability below 0.5 means the - model leans false, so it can never count as acceptance however clean a validation - sweep looks. Escalate is the largest lower threshold of at least ``ESCALATE_FLOOR`` + selected label lacks majority probability, so it can never count as acceptance + however clean a validation sweep looks. Escalate is the largest lower threshold + of at least ``ESCALATE_FLOOR`` with at most 0.1 of all positives below it. Fallbacks preserve ``escalate < accept``. """ diff --git a/tools/jev-optimize/jev_optimize/optimize.py b/tools/jev-optimize/jev_optimize/optimize.py index 01ff03f..86a3c97 100644 --- a/tools/jev-optimize/jev_optimize/optimize.py +++ b/tools/jev-optimize/jev_optimize/optimize.py @@ -8,10 +8,13 @@ from typing import Any import dspy +from dspy.experimental import ReAnchor -from .adapter import JevPrograms +from .adapter import program as make_program from .data import DatasetRow, load_jsonl, split_rows from .metrics import ( + ACCEPT_FLOOR, + ESCALATE_FLOOR, accuracy, brier_score, choose_thresholds, @@ -21,6 +24,11 @@ from .proposer import JevProposer from .questions import SETS, SPECS +# One wrong automatic decision costs this many right ones: 9 targets ~10% error +# in the accepted and rejected bands (owner decision 2026-09-27; 19 gave ~5% +# and sent most rows of the weak rules questions to the chat model). +WRONG_DECISION_PENALTY = 9.0 + def _inputs(row: DatasetRow) -> dict[str, Any]: return row.model_dump(exclude={"id", "set", "label", "source"}) @@ -41,13 +49,14 @@ def metric( if question_id not in example.label: raise ValueError(f"example does not carry label for {question_id}") label = example.label[question_id] + decision = prediction.decision if isinstance(label, bool): - probability = float(prediction.probability) - correct = (probability >= 0.5) == label + probability = float(decision.probability) + correct = decision.value == label feedback = f"label={str(label).lower()}; predicted_probability={probability:.3f}." else: - correct = prediction.choice == label - probability = float(prediction.probabilities.get(label, 0.0)) + correct = decision.value == label + probability = float(decision.probabilities.get(label, 0.0)) feedback = f"label={label}; predicted_label_probability={probability:.3f}." if feedback_includes_text and hasattr(example, "paragraph_text"): feedback += f" Synthetic/public context: {example.paragraph_text}" @@ -83,17 +92,18 @@ def evaluate_question( predictions = list(pool.map(lambda row: program(**_inputs(row)), selected)) for row, prediction in zip(selected, predictions, strict=True): label = row.label[question_id] + decision = prediction.decision if isinstance(label, bool): labels.append(label) - probabilities.append(float(prediction.probability)) + probabilities.append(float(decision.probability)) else: - correct = prediction.choice == label + correct = decision.value == label labels.append(correct) - probabilities.append(float(prediction.confidence)) + probabilities.append(float(decision.confidence)) multiclass_brier.append( sum( ( - float(prediction.probabilities.get(option, 0.0)) + float(decision.probabilities.get(option, 0.0)) - float(option == label) ) ** 2 @@ -149,6 +159,64 @@ def evaluate_question( return result +class _ReAnchorProgram(dspy.Module): + """Give ReAnchor a named predictor while the public factory stays a bare Predict.""" + + def __init__(self, predictor: dspy.Predict) -> None: + super().__init__() + self.predict = predictor + + def forward(self, **kwargs: Any) -> dspy.Prediction: + return self.predict(**kwargs) + + +def _reanchor_thresholds( + question_id: str, + predictor: dspy.Predict, + trainset: list[dspy.Example], + valset: list[dspy.Example], +) -> dict[str, float | bool]: + labels = [example.label[question_id] for example in [*trainset, *valset]] + positives = sum(label is True for label in labels) + negatives = sum(label is False for label in labels) + if positives < 20 or negatives < 20: + return {"accept": 0.7, "escalate": 0.3, "insufficient_data": True} + + def metric(*, false_positive_penalty: bool): + def score(example: dspy.Example, prediction: dspy.Prediction) -> float: + actual = bool(example.label[question_id]) + predicted = bool(prediction.decision.value) + if predicted == actual: + return 1.0 + penalized = ( + predicted and not actual + if false_positive_penalty + else actual and not predicted + ) + return -WRONG_DECISION_PENALTY if penalized else 0.0 + + return score + + wrapped = _ReAnchorProgram(predictor) + accept_program = ReAnchor(metric(false_positive_penalty=True)).compile( + wrapped, trainset=trainset, valset=valset + ) + escalate_program = ReAnchor(metric(false_positive_penalty=False)).compile( + wrapped, trainset=trainset, valset=valset + ) + fitted_accept = float( + accept_program.predict.fields.get("decision", {}).get("threshold", 0.5) + ) + fitted_escalate = float( + escalate_program.predict.fields.get("decision", {}).get("threshold", 0.5) + ) + accept = max(fitted_accept, ACCEPT_FLOOR) + escalate = max(fitted_escalate, ESCALATE_FLOOR) + if escalate >= accept: + escalate = max(ESCALATE_FLOOR, round(accept - 0.1, 2)) + return {"accept": accept, "escalate": escalate, "insufficient_data": False} + + def gated_question_update( question_id: str, *, @@ -161,11 +229,10 @@ def gated_question_update( ) -> dict[str, Any]: """Keep tuned wording only when validation and held-out accuracy do not regress. - Thresholds come from ``calibration`` when supplied: the selected wording - evaluated on train plus validation, which holds enough of each class for - the chooser where a validation split alone does not (closure outcomes are - 20% positive). Without it the selected validation sweep is used. The - held-out test split never feeds threshold choice. + Calibration uses train plus validation, which holds enough of each class + where validation alone does not. Noul questions run two ReAnchor passes; + Choice questions retain the confidence sweep. The held-out test split + never feeds threshold choice. """ baseline_validation_score = float(baseline_validation["accuracy"]) @@ -177,7 +244,22 @@ def gated_question_update( and tuned_test_accuracy >= baseline_test_accuracy ) selected_validation = tuned_validation if keep_tuned else baseline_validation - thresholds = choose_thresholds((calibration or selected_validation)["sweep"]) + calibration_data = calibration or selected_validation + if SPECS[question_id].kind == "choice": + thresholds = choose_thresholds(calibration_data["sweep"]) + else: + sweep = calibration_data["sweep"] + positives = int(sweep[0].get("positives", 0)) if sweep else 0 + negatives = int(sweep[0].get("negatives", 0)) if sweep else 0 + if positives < 20 or negatives < 20: + thresholds = {"accept": 0.7, "escalate": 0.3, "insufficient_data": True} + else: + thresholds = _reanchor_thresholds( + question_id, + calibration_data["program"], + calibration_data["trainset"], + calibration_data["valset"], + ) update: dict[str, Any] = { "instructions": ( tuned_instructions @@ -212,17 +294,18 @@ def optimize_set( *, budget: str = "light", feedback_includes_text: bool = False, + reflection_lm: Any | None = None, ) -> dict[str, Any]: rows = load_jsonl(data_path) - reflection_model = os.environ.get("OPENROUTER_REFLECTION_MODEL") - if not reflection_model: - raise RuntimeError("OPENROUTER_REFLECTION_MODEL is not set") - reflection_lm = dspy.LM( - f"openrouter/{reflection_model}", - api_base="https://openrouter.ai/api/v1", - extra_body={"provider": {"zdr": True}}, - ) - adapter = JevPrograms(client) + if reflection_lm is None: + reflection_model = os.environ.get("OPENROUTER_REFLECTION_MODEL") + if not reflection_model: + raise RuntimeError("OPENROUTER_REFLECTION_MODEL is not set") + reflection_lm = dspy.LM( + f"openrouter/{reflection_model}", + api_base="https://openrouter.ai/api/v1", + extra_body={"provider": {"zdr": True}}, + ) output: dict[str, Any] = { "model": getattr(client, "model", "typesafe/jev-1.13"), "dataset_sizes": {}, @@ -235,7 +318,7 @@ def optimize_set( output["dataset_sizes"][question_id] = { name: len(values) for name, values in parts.items() } - program = adapter.program(question_id) + program = make_program(question_id, client) baseline_validation = evaluate_question(program, question_id, parts["validation"]) baseline_test = evaluate_question(program, question_id, parts["test"]) spec = SPECS[question_id] @@ -269,20 +352,30 @@ def optimize_set( tuned_validation["accuracy"] >= baseline_validation["accuracy"] and tuned_test["accuracy"] >= baseline_test["accuracy"] ) + shipping_program = optimized if keep_tuned else program # Thresholds are chosen on train + validation for the wording that # will ship, so the sweep holds enough of each class; test stays out. calibration = evaluate_question( - optimized if keep_tuned else program, + shipping_program, question_id, [*parts["train"], *parts["validation"]], ) + calibration.update( + { + # ReAnchor must calibrate the predictor whose wording ships, + # using the same keep_tuned gate as the accuracy numbers. + "program": shipping_program, + "trainset": _examples(parts["train"], question_id), + "valset": _examples(parts["validation"], question_id), + } + ) update = gated_question_update( question_id, baseline_validation=baseline_validation, tuned_validation=tuned_validation, baseline_test=baseline_test, tuned_test=tuned_test, - tuned_instructions=optimized.predict.signature.instructions, + tuned_instructions=optimized.signature.instructions, calibration=calibration, ) update["calibration_rows"] = len(parts["train"]) + len(parts["validation"]) diff --git a/tools/jev-optimize/jev_optimize/questions.py b/tools/jev-optimize/jev_optimize/questions.py index fa578e0..f8cd224 100644 --- a/tools/jev-optimize/jev_optimize/questions.py +++ b/tools/jev-optimize/jev_optimize/questions.py @@ -6,6 +6,7 @@ from typing import Any import dspy +from dspy.experimental import Choice, Noul @dataclass(frozen=True) @@ -24,7 +25,7 @@ class ClosureFulfilled(dspy.Signature): obligation: dict[str, str] = dspy.InputField() later: dict[str, Any] = dspy.InputField() - probability: float = dspy.OutputField(desc="Probability that the obligation is fulfilled.") + decision: Noul = dspy.OutputField(desc="Probability that the obligation is fulfilled.") class ClosureWithdrawn(dspy.Signature): @@ -32,7 +33,7 @@ class ClosureWithdrawn(dspy.Signature): obligation: dict[str, str] = dspy.InputField() later: dict[str, Any] = dspy.InputField() - probability: float = dspy.OutputField(desc="Probability that the obligation is withdrawn.") + decision: Noul = dspy.OutputField(desc="Probability that the obligation is withdrawn.") class ClosureDeadlineChanged(dspy.Signature): @@ -40,7 +41,7 @@ class ClosureDeadlineChanged(dspy.Signature): obligation: dict[str, str] = dspy.InputField() later: dict[str, Any] = dspy.InputField() - probability: float = dspy.OutputField(desc="Probability that the deadline changed.") + decision: Noul = dspy.OutputField(desc="Probability that the deadline changed.") class ClosureModified(dspy.Signature): @@ -48,7 +49,17 @@ class ClosureModified(dspy.Signature): obligation: dict[str, str] = dspy.InputField() later: dict[str, Any] = dspy.InputField() - probability: float = dspy.OutputField(desc="Probability that the obligation was modified.") + decision: Noul = dspy.OutputField(desc="Probability that the obligation was modified.") + + +_CLOSURE_OUTCOME_OPTIONS = { + "fulfilled": "The later paragraph says the obligation was carried out.", + "withdrawn": "The later paragraph cancels or withdraws the obligation.", + "deadline_changed": "The later paragraph sets a different deadline for the obligation.", + "modified": "The later paragraph changes what the obligation requires.", + "none": "The later paragraph expresses none of the listed outcomes for the obligation.", +} +ClosureOutcomeDecision = Choice[tuple(_CLOSURE_OUTCOME_OPTIONS.items())] class ClosureOutcome(dspy.Signature): @@ -56,11 +67,9 @@ class ClosureOutcome(dspy.Signature): obligation: dict[str, str] = dspy.InputField() later: dict[str, Any] = dspy.InputField() - choice: str = dspy.OutputField(desc="The single expressed obligation outcome, or none.") - probabilities: dict[str, float] = dspy.OutputField( - desc="Probability for each issued obligation-outcome option." + decision: ClosureOutcomeDecision = dspy.OutputField( + desc="The single expressed obligation outcome, or none." ) - confidence: float = dspy.OutputField(desc="Confidence in the selected obligation outcome.") class TriageBase(dspy.Signature): @@ -72,47 +81,56 @@ class TriageBase(dspy.Signature): class TriageAsksRecipient(TriageBase): """The paragraph asks its recipient to do something.""" - probability: float = dspy.OutputField(desc="Probability of a request to the recipient.") + decision: Noul = dspy.OutputField(desc="Probability of a request to the recipient.") class TriageCommitsSender(TriageBase): """The paragraph commits its sender to doing something.""" - probability: float = dspy.OutputField(desc="Probability of a commitment by the sender.") + decision: Noul = dspy.OutputField(desc="Probability of a commitment by the sender.") class TriageAsksQuestion(TriageBase): """The paragraph asks a genuine question that seeks an answer.""" - probability: float = dspy.OutputField(desc="Probability of a genuine question.") + decision: Noul = dspy.OutputField(desc="Probability of a genuine question.") class TriageNamesTime(TriageBase): """The paragraph names a time, date, deadline, or event-relative time.""" - probability: float = dspy.OutputField(desc="Probability that a time is named.") + decision: Noul = dspy.OutputField(desc="Probability that a time is named.") class TriageBoilerplate(TriageBase): """The paragraph is a signature, legal footer, unsubscribe notice, or disclaimer.""" - probability: float = dspy.OutputField(desc="Probability that the paragraph is boilerplate.") + decision: Noul = dspy.OutputField(desc="Probability that the paragraph is boilerplate.") class TriageAutomatedNotification(TriageBase): """The paragraph is an automatically generated notification.""" - probability: float = dspy.OutputField(desc="Probability of an automated notification.") + decision: Noul = dspy.OutputField(desc="Probability of an automated notification.") + + +_EXTRACT_CLAIM_TYPE_OPTIONS = { + "request": "The paragraph asks its recipient to do something.", + "promise": "The paragraph commits its sender to doing something.", + "question": "The paragraph asks a genuine question that seeks an answer.", + "attribution": "The paragraph attributes an obligation or commitment to another party.", + "delegation": "The paragraph delegates an obligation from one party to another.", + "none": "The paragraph expresses none of the listed claim types.", +} +ExtractClaimTypeDecision = Choice[tuple(_EXTRACT_CLAIM_TYPE_OPTIONS.items())] class ExtractClaimType(TriageBase): """Which single claim type, if any, does the paragraph express?""" - choice: str = dspy.OutputField(desc="The single expressed claim type, or none.") - probabilities: dict[str, float] = dspy.OutputField( - desc="Probability for each issued claim-type option." + decision: ExtractClaimTypeDecision = dspy.OutputField( + desc="The single expressed claim type, or none." ) - confidence: float = dspy.OutputField(desc="Confidence in the selected claim type.") # `extract.waiting_party` and `extract.temporal` (P5) are dynamic-option @@ -129,34 +147,40 @@ class ExtractClaimType(TriageBase): # this bootstrap text, but this harness has no labeled corpus for either # question yet -- see the design doc's P5 section for what a corpus would # need. +_EXTRACT_WAITING_PARTY_OPTIONS = { + "none": "No participant is waiting on the signed-in user for this paragraph." +} +ExtractWaitingPartyDecision = Choice[tuple(_EXTRACT_WAITING_PARTY_OPTIONS.items())] + + class ExtractWaitingParty(TriageBase): """Which participant is waiting on the signed-in user for this paragraph?""" participants: list[dict[str, str]] = dspy.InputField() - choice: str = dspy.OutputField(desc="The waiting participant's issued handle, or none.") - probabilities: dict[str, float] = dspy.OutputField( - desc="Probability for each issued participant-handle option, plus none." + decision: ExtractWaitingPartyDecision = dspy.OutputField( + desc="The waiting participant's issued handle, or none." ) - confidence: float = dspy.OutputField(desc="Confidence in the selected waiting party.") + + +_EXTRACT_TEMPORAL_OPTIONS = { + "none": "No temporal candidate applies to this paragraph." +} +ExtractTemporalDecision = Choice[tuple(_EXTRACT_TEMPORAL_OPTIONS.items())] class ExtractTemporal(TriageBase): """Which named time, date, or deadline candidate, if any, does this paragraph express?""" - choice: str = dspy.OutputField( + decision: ExtractTemporalDecision = dspy.OutputField( desc="The chosen normalized temporal candidate's issued handle, or none." ) - probabilities: dict[str, float] = dspy.OutputField( - desc="Probability for each issued temporal-candidate option, plus none." - ) - confidence: float = dspy.OutputField(desc="Confidence in the selected temporal candidate.") def _rule_signature(name: str, instructions: str, fields: tuple[str, ...]) -> type[dspy.Signature]: - annotations = {field: str for field in fields} | {"probability": float} + annotations = {field: str for field in fields} | {"decision": Noul} namespace = {"__annotations__": annotations, "__doc__": instructions} namespace.update({field: dspy.InputField() for field in fields}) - namespace["probability"] = dspy.OutputField(desc=f"Probability that {instructions.lower()}") + namespace["decision"] = dspy.OutputField(desc=f"Probability that {instructions.lower()}") return type(name, (dspy.Signature,), namespace) @@ -183,17 +207,21 @@ def _rule_signature(name: str, instructions: str, fields: tuple[str, ...]) -> ty ) +_RULES_DEADLINE_KIND_OPTIONS = { + "event_tied": "The deadline is tied to a named event.", + "soft": "The timing is flexible or aspirational.", + "unknown": "The phrase has no supported deadline classification.", +} +RulesDeadlineKindDecision = Choice[tuple(_RULES_DEADLINE_KIND_OPTIONS.items())] + + class RulesDeadlineKind(dspy.Signature): """Classify how the phrase expresses a deadline.""" phrase: str = dspy.InputField() - choice: str = dspy.OutputField( + decision: RulesDeadlineKindDecision = dspy.OutputField( desc="event_tied: tied to an event; soft: flexible timing; unknown: neither classification." ) - probabilities: dict[str, float] = dspy.OutputField( - desc="Probability for each issued deadline-kind option." - ) - confidence: float = dspy.OutputField(desc="Confidence in the selected deadline kind.") _CLOSURE_FIELDS = ( @@ -234,13 +262,7 @@ class RulesDeadlineKind(dspy.Signature): ("later.paragraph_text",), _CLOSURE_FIELDS, "choice", - { - "fulfilled": "The later paragraph says the obligation was carried out.", - "withdrawn": "The later paragraph cancels or withdraws the obligation.", - "deadline_changed": "The later paragraph sets a different deadline for the obligation.", - "modified": "The later paragraph changes what the obligation requires.", - "none": "The later paragraph expresses none of the listed outcomes for the obligation.", - }, + _CLOSURE_OUTCOME_OPTIONS, ), "triage.asks_recipient": QuestionSpec( "triage", TriageAsksRecipient, @@ -279,14 +301,7 @@ class RulesDeadlineKind(dspy.Signature): ("paragraph_text",), _TRIAGE_FIELDS, "choice", - { - "request": "The paragraph asks its recipient to do something.", - "promise": "The paragraph commits its sender to doing something.", - "question": "The paragraph asks a genuine question that seeks an answer.", - "attribution": "The paragraph attributes an obligation or commitment to another party.", - "delegation": "The paragraph delegates an obligation from one party to another.", - "none": "The paragraph expresses none of the listed claim types.", - }, + _EXTRACT_CLAIM_TYPE_OPTIONS, ), "extract.waiting_party": QuestionSpec( "extract", @@ -295,7 +310,7 @@ class RulesDeadlineKind(dspy.Signature): ("paragraph_text", "participants"), (*_TRIAGE_FIELDS, "participants"), "choice", - {"none": "No participant is waiting on the signed-in user for this paragraph."}, + _EXTRACT_WAITING_PARTY_OPTIONS, ), "extract.temporal": QuestionSpec( "extract", @@ -304,7 +319,7 @@ class RulesDeadlineKind(dspy.Signature): ("paragraph_text",), _TRIAGE_FIELDS, "choice", - {"none": "No temporal candidate applies to this paragraph."}, + _EXTRACT_TEMPORAL_OPTIONS, ), "rules.recap": QuestionSpec( "rules", RulesRecap, @@ -339,11 +354,7 @@ class RulesDeadlineKind(dspy.Signature): ("phrase",), ("phrase",), "choice", - { - "event_tied": "The deadline is tied to a named event.", - "soft": "The timing is flexible or aspirational.", - "unknown": "The phrase has no supported deadline classification.", - }, + _RULES_DEADLINE_KIND_OPTIONS, ), } @@ -352,14 +363,3 @@ class RulesDeadlineKind(dspy.Signature): name: tuple(key for key, spec in SPECS.items() if spec.set_name == name) for name in ("closure", "triage", "rules", "extract") } - - -def jev_question(question_id: str) -> dict[str, Any]: - spec = SPECS[question_id] - question: dict[str, Any] = { - "type": spec.kind, - "instructions": (spec.signature.__doc__ or "").strip(), - } - if spec.options: - question["criteria"] = spec.options - return question diff --git a/tools/jev-optimize/pyproject.toml b/tools/jev-optimize/pyproject.toml index 35d96ee..95a5bf5 100644 --- a/tools/jev-optimize/pyproject.toml +++ b/tools/jev-optimize/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "Offline-first DSPy harness for OpenLoops Jev question optimization" requires-python = ">=3.11" dependencies = [ - "dspy==3.3.1", + "dspy==3.4.0", "httpx", "numpy", "pydantic", diff --git a/tools/jev-optimize/tests/test_client.py b/tools/jev-optimize/tests/test_client.py index 6c0eb64..9c7695a 100644 --- a/tools/jev-optimize/tests/test_client.py +++ b/tools/jev-optimize/tests/test_client.py @@ -5,7 +5,7 @@ import httpx import pytest -from jev_optimize.adapter import JevPrograms +from jev_optimize.adapter import JevLM, program from jev_optimize.client import ( DecisionResponseError, DecisionsClient, @@ -97,7 +97,7 @@ def test_replay_uses_stable_state_and_question_hash(): assert request_bytes(MODEL, state, QUESTION, zdr_member=False).endswith(b"}") -def test_one_signature_becomes_one_exact_typed_question(): +def test_one_signature_becomes_one_exact_typed_question(monkeypatch): class CaptureClient: def __init__(self): self.call = None @@ -107,10 +107,13 @@ def decide(self, state, questions): return {"triage.asks_recipient": {"type": "noul", "noul": 0.75}} client = CaptureClient() - prediction = JevPrograms(client).program("triage.asks_recipient")( + monkeypatch.setattr("dspy.cache.get", lambda _request: None) + monkeypatch.setattr("dspy.cache.put", lambda _request, _response: None) + prediction = program("triage.asks_recipient", client)( subject="Mosaic update", paragraph_text="Please review the draft.", from_user=False ) - assert prediction.probability == 0.75 + assert prediction.decision.value is True + assert prediction.decision.probability == 0.75 assert client.call == ( { "subject": "Mosaic update", @@ -126,13 +129,109 @@ def decide(self, state, questions): ) -def test_jev_predict_deepcopy_shares_client_without_warning(): +def test_choice_signature_decodes_one_native_decision(monkeypatch): + class ChoiceClient: + def __init__(self): + self.questions = None + + def decide(self, state, questions): + self.questions = questions + return { + "closure.outcome": { + "type": "choice", + "choice": "fulfilled", + "probabilities": { + "fulfilled": 0.8, + "withdrawn": 0.0, + "deadline_changed": 0.0, + "modified": 0.0, + "none": 0.2, + }, + "confidence": 0.8, + } + } + + monkeypatch.setattr("dspy.cache.get", lambda _request: None) + monkeypatch.setattr("dspy.cache.put", lambda _request, _response: None) + client = ChoiceClient() + prediction = program("closure.outcome", client)( + obligation={"title": "Synthetic", "evidence_text": "Please send it."}, + later={"paragraph_text": "Sent.", "from_user": True, "days_later": 1}, + ) + assert prediction.decision.value == "fulfilled" + assert prediction.decision.probabilities["fulfilled"] == 0.8 + assert prediction.decision.confidence == 0.8 + assert client.questions["closure.outcome"]["type"] == "choice" + assert set(client.questions["closure.outcome"]["criteria"]) == { + "fulfilled", + "withdrawn", + "deadline_changed", + "modified", + "none", + } + + +def test_jev_lm_deepcopy_shares_client_without_warning(): client = object() - program = JevPrograms(client).program("triage.asks_recipient") + lm = JevLM(client, "triage.asks_recipient") with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - copied = deepcopy(program) - assert copied is not program - assert copied.adapter is not program.adapter - assert copied.adapter.client is client + copied = deepcopy(lm) + assert copied is not lm + assert copied.client is client + assert copied.question_id == "triage.asks_recipient" assert not [warning for warning in caught if "Failed to deep copy" in str(warning.message)] + + +def test_jev_lm_caches_identical_decision_requests(monkeypatch): + calls = 0 + + class CountingClient: + def decide(self, state, questions): + nonlocal calls + calls += 1 + return {"triage.asks_recipient": {"type": "noul", "noul": 0.6}} + + cached = {} + cache_requests = [] + + def cache_get(request): + cache_requests.append(request) + return cached.get(repr(request)) + + monkeypatch.setattr("dspy.cache.get", cache_get) + monkeypatch.setattr( + "dspy.cache.put", + lambda request, response: cached.__setitem__(repr(request), response), + ) + lm = JevLM(CountingClient(), "triage.asks_recipient") + state = { + "instructions": "Synthetic decision.", + "input_fields": "1. `text` (str):", + "inputs": {"text": "Cache-specific synthetic state."}, + } + questions = {"decision": {"type": "noul", "instructions": "Output description."}} + assert lm(state, questions) == lm(deepcopy(state), deepcopy(questions)) + assert calls == 1 + assert cache_requests[0] == { + "provider": "jev", + "state": {"text": "Cache-specific synthetic state."}, + "questions": { + "triage.asks_recipient": { + "type": "noul", + "instructions": "Synthetic decision.", + } + }, + } + + +def test_jev_lm_rejects_multiple_decision_questions(): + lm = JevLM(object(), "triage.asks_recipient") + with pytest.raises(ValueError, match="exactly one 'decision'"): + lm( + {"instructions": "Synthetic.", "inputs": {"text": "Synthetic."}}, + { + "decision": {"type": "noul", "instructions": "Synthetic."}, + "other": {"type": "noul", "instructions": "Other."}, + }, + ) diff --git a/tools/jev-optimize/tests/test_data_metrics.py b/tools/jev-optimize/tests/test_data_metrics.py index b5233c5..5c9b834 100644 --- a/tools/jev-optimize/tests/test_data_metrics.py +++ b/tools/jev-optimize/tests/test_data_metrics.py @@ -76,7 +76,7 @@ def test_evaluate_reports_confusion_at_registry_thresholds(): probabilities = iter((0.9, 0.6, 0.4, 0.1)) def program(**_inputs): - return SimpleNamespace(probability=next(probabilities)) + return SimpleNamespace(decision=SimpleNamespace(probability=next(probabilities))) rows = [ DatasetRow( @@ -117,7 +117,7 @@ def test_evaluate_skips_rows_without_the_question_label(): def program(**inputs): calls.append(inputs) - return SimpleNamespace(probability=0.9) + return SimpleNamespace(decision=SimpleNamespace(probability=0.9)) result = evaluate_question(program, "triage.asks_recipient", rows) assert result["accuracy"] == 1.0 diff --git a/tools/jev-optimize/tests/test_offline_end_to_end.py b/tools/jev-optimize/tests/test_offline_end_to_end.py new file mode 100644 index 0000000..4bc3291 --- /dev/null +++ b/tools/jev-optimize/tests/test_offline_end_to_end.py @@ -0,0 +1,290 @@ +import hashlib +import json +import re +import shutil +import threading +from pathlib import Path + +import dspy +import httpx +import pytest +from dspy.clients import configure_cache +from dspy.utils.dummies import DummyLM + +import jev_optimize.optimize as optimize_module +from jev_optimize.adapter import program +from jev_optimize.client import DecisionResponseError, DecisionsClient +from jev_optimize.data import DatasetRow, load_jsonl, split_name, write_jsonl +from jev_optimize.metrics import ACCEPT_FLOOR, ESCALATE_FLOOR +from jev_optimize.optimize import evaluate_question, optimize_set +from jev_optimize.questions import SETS, SPECS +from jev_optimize.registry import DEFAULT_REGISTRY, merge_registry + +TOOL_ROOT = Path(__file__).parents[1] +MODEL = "typesafe/jev-1.13" +REPORTED_MODEL = "typesafe/jev-1.13-20260917" + + +def _stable_jitter(question_id, text): + digest = hashlib.sha256(f"{question_id}\0{text}".encode()).digest() + unit = int.from_bytes(digest[:8], "big") / ((1 << 64) - 1) + return 0.3 * unit - 0.15 + + +def _noul_probability(question_id, text): + lower = text.casefold() + signals = { + "triage.asks_recipient": ("please", "could you", "would you", "can you"), + "triage.commits_sender": ("i will", "i'll", "we will", "we'll"), + "triage.asks_question": ("?",), + "triage.names_time": ( + "today", + "tomorrow", + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "deadline", + ), + "triage.boilerplate": ("unsubscribe", "confidential", "disclaimer"), + "triage.automated_notification": ("automated", "do not reply", "notification"), + } + fallback = ("done", "sent", "completed") + matched = any(token in lower for token in signals.get(question_id, fallback)) + return min(0.99, max(0.01, (0.62 if matched else 0.38) + _stable_jitter(question_id, text))) + + +def _choice_answer(question_id, text, criteria): + labels = list(criteria) + weights = [] + for label in labels: + digest = hashlib.sha256(f"{question_id}\0{label}\0{text}".encode()).digest() + weight = 0.2 + int.from_bytes(digest[:4], "big") / (1 << 32) + if label.replace("_", " ") in text.casefold(): + weight += 1.0 + weights.append(weight) + total = sum(weights) + probabilities = {label: weight / total for label, weight in zip(labels, weights, strict=True)} + probabilities[labels[-1]] = 1.0 - sum(probabilities[label] for label in labels[:-1]) + choice = max(labels, key=probabilities.get) + return { + "type": "choice", + "choice": choice, + "probabilities": probabilities, + "confidence": probabilities[choice], + } + + +class FakeDecisionsEndpoint: + def __init__(self, mode="normal"): + self.mode = mode + self.calls = 0 + self.question_ids = set() + self.saw_envelope_state = False + self._lock = threading.Lock() + + def __call__(self, request): + assert request.headers.get("Authorization") == "Bearer test-key-not-real" + body = json.loads(request.content) + assert set(body) == {"model", "state", "questions", "provider"} + assert body["model"] == MODEL + assert body["provider"] == {"zdr": True} + assert "inputs" not in body["state"] + assert "instructions" not in body["state"] + assert len(body["questions"]) == 1 + question_id, question = next(iter(body["questions"].items())) + assert re.fullmatch(r"[a-z_]+\.[a-z_]+", question_id) + assert set(body["state"]) == set(SPECS[question_id].signature.input_fields) + expected_question_keys = {"type", "instructions"} + if SPECS[question_id].options: + expected_question_keys.add("criteria") + assert set(question) == expected_question_keys + assert question["type"] == SPECS[question_id].kind + with self._lock: + self.calls += 1 + call_number = self.calls + self.question_ids.add(question_id) + self.saw_envelope_state |= "inputs" in body["state"] + if self.mode == "http_400": + return httpx.Response(400, json={"error": "synthetic rejection"}) + if self.mode == "retry" and call_number == 1: + return httpx.Response(429, json={"error": "synthetic rate limit"}) + text = json.dumps(body["state"], ensure_ascii=False, sort_keys=True) + if question["type"] == "noul": + probability = _noul_probability(question_id, text) + if self.mode == "malformed" and call_number == 1: + probability = 1.5 + answer = {"type": "noul", "noul": probability} + else: + answer = _choice_answer(question_id, text, question["criteria"]) + assert sum(answer["probabilities"].values()) == pytest.approx(1.0) + return httpx.Response( + 200, + json={ + "model": REPORTED_MODEL, + "usage": {"input_tokens": max(1, len(request.content) // 4), "cost": 0.0001}, + "answers": {question_id: answer}, + }, + ) + + +def _reduced_triage_rows(): + rows = load_jsonl(TOOL_ROOT / "data" / "synthetic" / "triage.jsonl") + selected = [] + for question_id in SETS["triage"]: + prefix = [] + for row in (candidate for candidate in rows if question_id in candidate.label): + prefix.append(row) + train_validation = [ + item for item in prefix if split_name(item.id, small=True) != "test" + ] + positives = sum(item.label[question_id] is True for item in train_validation) + negatives = sum(item.label[question_id] is False for item in train_validation) + split_counts = { + name: sum(split_name(item.id, small=True) == name for item in prefix) + for name in ("train", "validation", "test") + } + if positives >= 20 and negatives >= 20 and all(split_counts.values()): + break + assert positives >= 20 and negatives >= 20 + assert all(split_counts.values()) + selected.extend(prefix) + return selected + + +def _one_row(): + return DatasetRow( + id="offline-evaluation-row", + set="triage", + subject="Synthetic request", + paragraph_text="Could you please review this synthetic note?", + from_user=False, + label={"triage.asks_recipient": True}, + source="synthetic-stub", + ) + + +def _client(endpoint): + return DecisionsClient( + model=MODEL, + transport=httpx.MockTransport(endpoint), + max_retries=2, + ) + + +@pytest.fixture(autouse=True) +def memory_only_dspy_cache(): + had_cache = "cache" in dspy.__dict__ + previous_cache = dspy.__dict__.get("cache") + configure_cache( + enable_disk_cache=False, + enable_memory_cache=True, + disk_cache_dir=None, + ) + yield + with dspy._cache_lock: + if had_cache: + dspy.cache = previous_cache + else: + dspy.__dict__.pop("cache", None) + + +def test_real_optimize_path_is_offline_and_production_shaped(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key-not-real") + data_path = tmp_path / "triage-reduced.jsonl" + write_jsonl(data_path, (row.model_dump() for row in _reduced_triage_rows())) + endpoint = FakeDecisionsEndpoint() + reflection_lm = DummyLM( + { + "paragraph_text": { + "reasoning": "The statement stays literal and names the primary field.", + "proposed_statement": "paragraph_text states the relevant triage condition.", + } + } + ) + real_reanchor = optimize_module.ReAnchor + reanchor_calls = [] + + class RecordingReAnchor(real_reanchor): + def compile(self, student, *, trainset, valset=None): + reanchor_calls.append((student, trainset, valset)) + return super().compile(student, trainset=trainset, valset=valset) + + monkeypatch.setattr(optimize_module, "ReAnchor", RecordingReAnchor) + result = optimize_set( + "triage", + _client(endpoint), + data_path, + budget="light", + reflection_lm=reflection_lm, + ) + + assert set(result["questions"]) == set(SETS["triage"]) + if "extract.claim_type" in SETS["triage"]: + assert "extract.claim_type" in result["questions"] + for question_id in SETS["triage"]: + update = result["questions"][question_id] + assert isinstance(update["accept"], float) + assert isinstance(update["escalate"], float) + assert ESCALATE_FLOOR <= update["escalate"] < update["accept"] + assert update["accept"] >= ACCEPT_FLOOR + assert update["insufficient_data"] is False + assert {"accuracy", "brier", "ece_10"} <= set( + result["test_metrics"][question_id] + ) + noul_count = sum(SPECS[question_id].kind == "noul" for question_id in SETS["triage"]) + assert len(reanchor_calls) == 2 * noul_count + assert endpoint.question_ids == set(SETS["triage"]) + assert endpoint.saw_envelope_state is False + + registry_path = tmp_path / "decision-questions.json" + shutil.copyfile(DEFAULT_REGISTRY, registry_path) + before = json.loads(registry_path.read_text(encoding="utf-8")) + digest = merge_registry( + {"triage": result}, + path=registry_path, + allow_baseline=True, + ) + after = json.loads(registry_path.read_text(encoding="utf-8")) + assert re.fullmatch(r"[0-9a-f]{64}", digest) + assert set(after) == set(before) + assert {key: set(value) for key, value in after["questions"].items()} == { + key: set(value) for key, value in before["questions"].items() + } + + +def test_evaluate_question_rejects_malformed_reply(monkeypatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key-not-real") + endpoint = FakeDecisionsEndpoint("malformed") + with pytest.raises(DecisionResponseError): + evaluate_question( + program("triage.asks_recipient", _client(endpoint)), + "triage.asks_recipient", + [_one_row()], + ) + + +def test_evaluate_question_propagates_rejected_row(monkeypatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key-not-real") + endpoint = FakeDecisionsEndpoint("http_400") + with pytest.raises(httpx.HTTPStatusError): + evaluate_question( + program("triage.asks_recipient", _client(endpoint)), + "triage.asks_recipient", + [_one_row()], + ) + + +def test_evaluate_question_retries_429_then_succeeds(monkeypatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "test-key-not-real") + monkeypatch.setattr("jev_optimize.client.time.sleep", lambda _seconds: None) + endpoint = FakeDecisionsEndpoint("retry") + result = evaluate_question( + program("triage.asks_recipient", _client(endpoint)), + "triage.asks_recipient", + [_one_row()], + ) + assert endpoint.calls == 2 + assert {"accuracy", "brier", "ece_10"} <= set(result) diff --git a/tools/jev-optimize/tests/test_proposer_optimize.py b/tools/jev-optimize/tests/test_proposer_optimize.py index 3937734..3c10f31 100644 --- a/tools/jev-optimize/tests/test_proposer_optimize.py +++ b/tools/jev-optimize/tests/test_proposer_optimize.py @@ -1,9 +1,15 @@ from types import SimpleNamespace +import dspy import pytest +from dspy.experimental import Noul -from jev_optimize.metrics import threshold_sweep -from jev_optimize.optimize import gated_question_update +from jev_optimize.metrics import ACCEPT_FLOOR, ESCALATE_FLOOR, threshold_sweep +from jev_optimize.optimize import ( + _reanchor_thresholds, + gated_question_update, + optimize_set, +) from jev_optimize.proposer import JevProposer, validate_statement from jev_optimize.questions import SPECS @@ -23,6 +29,13 @@ def test_every_question_declares_intent_and_field_contract(): assert all(set(spec.primary_fields) <= set(spec.allowed_fields) for spec in SPECS.values()) +def test_every_question_has_one_native_decision_output(): + for spec in SPECS.values(): + assert list(spec.signature.output_fields) == ["decision"] + if spec.kind == "noul": + assert spec.signature.output_fields["decision"].annotation is Noul + + @pytest.mark.parametrize( ("statement", "expected"), [ @@ -110,8 +123,8 @@ def test_proposer_validator_enforces_field_contract(statement, expected): def _metrics(accuracy): - labels = [True] * 20 + [False] * 20 - probabilities = [0.9] * 20 + [0.1] * 20 + labels = [True] * 10 + [False] * 10 + probabilities = [0.9] * 10 + [0.1] * 10 return {"accuracy": accuracy, "sweep": threshold_sweep(labels, probabilities)} @@ -143,7 +156,7 @@ def test_result_gate_keeps_non_regressing_tuned_statement(): assert "kept_baseline" not in update -def test_result_gate_takes_thresholds_from_the_calibration_sweep_when_given(): +def test_result_gate_runs_reanchor_on_the_calibration_population_when_given(monkeypatch): # Validation alone has too few positives (10) for the chooser; the # calibration sweep (train + validation) has enough and is used instead. thin = { @@ -153,7 +166,17 @@ def test_result_gate_takes_thresholds_from_the_calibration_sweep_when_given(): calibration = { "accuracy": 0.9, "sweep": threshold_sweep([True] * 60 + [False] * 60, [0.9] * 60 + [0.1] * 60), + "program": object(), + "trainset": [object()], + "valset": [object()], } + seen = [] + + def fake_reanchor(question_id, predictor, trainset, valset): + seen.append((question_id, predictor, trainset, valset)) + return {"accept": 0.8, "escalate": 0.25, "insufficient_data": False} + + monkeypatch.setattr("jev_optimize.optimize._reanchor_thresholds", fake_reanchor) without = gated_question_update( "closure.fulfilled", baseline_validation=thin, @@ -173,4 +196,157 @@ def test_result_gate_takes_thresholds_from_the_calibration_sweep_when_given(): calibration=calibration, ) assert with_calibration["insufficient_data"] is False - assert with_calibration["accept"] >= 0.5 > with_calibration["escalate"] + assert with_calibration["accept"] == 0.8 + assert with_calibration["escalate"] == 0.25 + assert len(seen) == 1 + + +def test_reanchor_uses_mirrored_heavy_penalties_and_reads_wrapped_predict(monkeypatch): + metrics = [] + thresholds = iter((0.8, 0.25)) + + class FakeReAnchor: + def __init__(self, metric): + metrics.append(metric) + + def compile(self, program, *, trainset, valset): + assert program.predict is predictor + assert trainset and valset + return SimpleNamespace( + predict=SimpleNamespace(fields={"decision": {"threshold": next(thresholds)}}) + ) + + monkeypatch.setattr("jev_optimize.optimize.ReAnchor", FakeReAnchor) + predictor = object() + examples = [ + dspy.Example(label={"closure.fulfilled": value}) for value in [True] * 20 + [False] * 20 + ] + result = _reanchor_thresholds( + "closure.fulfilled", predictor, examples[:20], examples[20:] + ) + true_prediction = SimpleNamespace(decision=SimpleNamespace(value=True)) + false_prediction = SimpleNamespace(decision=SimpleNamespace(value=False)) + false_example = dspy.Example(label={"closure.fulfilled": False}) + true_example = dspy.Example(label={"closure.fulfilled": True}) + assert metrics[0](false_example, true_prediction) == -9.0 + assert metrics[0](true_example, false_prediction) == 0.0 + assert metrics[1](false_example, true_prediction) == 0.0 + assert metrics[1](true_example, false_prediction) == -9.0 + assert result == {"accept": 0.8, "escalate": 0.25, "insufficient_data": False} + + +def test_reanchor_defaults_to_dspy_threshold_when_calibration_restores_empty_fields( + monkeypatch, +): + class EmptyFieldsReAnchor: + def __init__(self, _metric): + pass + + def compile(self, _program, *, trainset, valset): + assert trainset and valset + return SimpleNamespace(predict=SimpleNamespace(fields={})) + + monkeypatch.setattr("jev_optimize.optimize.ReAnchor", EmptyFieldsReAnchor) + examples = [ + dspy.Example(label={"closure.fulfilled": value}) + for value in [True] * 20 + [False] * 20 + ] + + result = _reanchor_thresholds( + "closure.fulfilled", object(), examples[:20], examples[20:] + ) + + expected_accept = max(0.5, ACCEPT_FLOOR) + expected_escalate = max(0.5, ESCALATE_FLOOR) + if expected_escalate >= expected_accept: + expected_escalate = max(ESCALATE_FLOOR, round(expected_accept - 0.1, 2)) + assert result == { + "accept": expected_accept, + "escalate": expected_escalate, + "insufficient_data": False, + } + + +def test_optimize_set_calibrates_baseline_predictor_when_tuned_wording_is_rejected( + monkeypatch, tmp_path +): + question_id = "closure.fulfilled" + baseline = SimpleNamespace(signature=SimpleNamespace(instructions="baseline")) + optimized = SimpleNamespace(signature=SimpleNamespace(instructions="tuned")) + parts = {"train": [object()], "validation": [object()], "test": [object()]} + evaluation_calls = [] + evaluation_results = iter( + ( + {"accuracy": 0.9}, + {"accuracy": 0.9}, + {"accuracy": 0.8}, + {"accuracy": 0.9}, + { + "accuracy": 0.9, + "sweep": threshold_sweep( + [True] * 20 + [False] * 20, + [0.9] * 20 + [0.1] * 20, + ), + }, + ) + ) + + def fake_evaluate(program, _question_id, _rows): + evaluation_calls.append(program) + return next(evaluation_results) + + class FakeGEPA: + def __init__(self, **_kwargs): + pass + + def compile(self, program, *, trainset, valset): + assert program is baseline + assert trainset and valset + return optimized + + class FakeProposer: + def __init__(self, *_args, **_kwargs): + self.rejection_count = 0 + + calibrated_programs = [] + + def fake_reanchor(_question_id, predictor, _trainset, _valset): + calibrated_programs.append(predictor) + return {"accept": 0.7, "escalate": 0.3, "insufficient_data": False} + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("OPENROUTER_REFLECTION_MODEL", "synthetic-model") + monkeypatch.setattr("jev_optimize.optimize.SETS", {"test-set": (question_id,)}) + monkeypatch.setattr("jev_optimize.optimize.load_jsonl", lambda _path: [object()]) + monkeypatch.setattr("jev_optimize.optimize.split_rows", lambda _rows, **_kwargs: parts) + monkeypatch.setattr("jev_optimize.optimize.make_program", lambda *_args: baseline) + monkeypatch.setattr("jev_optimize.optimize.evaluate_question", fake_evaluate) + monkeypatch.setattr("jev_optimize.optimize._examples", lambda rows, _question_id: rows) + monkeypatch.setattr("jev_optimize.optimize.JevProposer", FakeProposer) + monkeypatch.setattr("jev_optimize.optimize.dspy.LM", lambda *_args, **_kwargs: object()) + monkeypatch.setattr("jev_optimize.optimize.dspy.GEPA", FakeGEPA) + monkeypatch.setattr("jev_optimize.optimize._reanchor_thresholds", fake_reanchor) + + result = optimize_set("test-set", SimpleNamespace(model="test-model"), "unused.jsonl") + + assert evaluation_calls[-1] is baseline + assert calibrated_programs == [baseline] + assert result["questions"][question_id]["kept_baseline"] is True + + +def test_choice_gate_keeps_using_confidence_sweep(): + metrics = { + "accuracy": 0.9, + "sweep": threshold_sweep([True] * 20 + [False] * 20, [0.9] * 20 + [0.1] * 20), + } + update = gated_question_update( + "closure.outcome", + baseline_validation=metrics, + tuned_validation=metrics, + baseline_test={"accuracy": 0.9}, + tuned_test={"accuracy": 0.9}, + tuned_instructions="later.paragraph_text states one obligation outcome.", + calibration=metrics, + ) + assert update["accept"] == 0.5 + assert update["escalate"] == 0.45 diff --git a/tools/jev-optimize/tests/test_wire_shape.py b/tools/jev-optimize/tests/test_wire_shape.py new file mode 100644 index 0000000..a73f8f4 --- /dev/null +++ b/tools/jev-optimize/tests/test_wire_shape.py @@ -0,0 +1,81 @@ +from pathlib import Path + +import pytest + +from jev_optimize.adapter import program +from jev_optimize.client import stable_request_hash +from jev_optimize.data import load_jsonl +from jev_optimize.questions import SPECS + +ROOT = Path(__file__).parents[1] + + +class CaptureClient: + def __init__(self, question_id): + self.question_id = question_id + self.calls = [] + + def decide(self, state, questions): + self.calls.append((state, questions)) + question = questions[self.question_id] + if question["type"] == "noul": + answer = {"type": "noul", "noul": 0.75} + else: + labels = list(question["criteria"]) + answer = { + "type": "choice", + "choice": labels[0], + "probabilities": { + label: float(index == 0) for index, label in enumerate(labels) + }, + "confidence": 1.0, + } + return {self.question_id: answer} + + +def _row_inputs(question_id): + spec = SPECS[question_id] + data_set = "triage" if spec.set_name == "extract" else spec.set_name + rows = load_jsonl(ROOT / "data" / "synthetic" / f"{data_set}.jsonl") + row = next( + (candidate for candidate in rows if question_id in candidate.label), + rows[0], + ) + values = row.model_dump(exclude={"id", "set", "label", "source"}) + # The bootstrap waiting-party question has no labeled corpus yet. + values.setdefault("participants", []) + return {name: values[name] for name in spec.signature.input_fields} + + +@pytest.mark.parametrize("question_id", SPECS) +def test_native_predict_sends_the_production_wire_shape(question_id, monkeypatch): + monkeypatch.setattr("dspy.cache.get", lambda _request: None) + monkeypatch.setattr("dspy.cache.put", lambda _request, _response: None) + spec = SPECS[question_id] + inputs = _row_inputs(question_id) + expected_question = { + "type": spec.kind, + "instructions": spec.signature.instructions, + **({"criteria": spec.options} if spec.options else {}), + } + expected = (inputs, {question_id: expected_question}) + client = CaptureClient(question_id) + + program(question_id, client)(**inputs) + + captured = client.calls[0] + assert captured == expected + assert stable_request_hash(*captured) == stable_request_hash(*expected) + + +def test_gepa_instruction_update_moves_to_production_question(monkeypatch): + monkeypatch.setattr("dspy.cache.get", lambda _request: None) + monkeypatch.setattr("dspy.cache.put", lambda _request, _response: None) + question_id = "triage.asks_recipient" + client = CaptureClient(question_id) + predictor = program(question_id, client) + predictor.signature = predictor.signature.with_instructions("New wording.") + + predictor(**_row_inputs(question_id)) + + assert client.calls[0][1][question_id]["instructions"] == "New wording." diff --git a/tools/jev-optimize/uv.lock b/tools/jev-optimize/uv.lock index 8a42634..f2b5139 100644 --- a/tools/jev-optimize/uv.lock +++ b/tools/jev-optimize/uv.lock @@ -417,7 +417,7 @@ wheels = [ [[package]] name = "dspy" -version = "3.3.1" +version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -435,9 +435,9 @@ dependencies = [ { name = "tenacity" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/47/36/cafd7ee235481f299426243d9c3608b9fa2e64227940cce50f94feea9570/dspy-3.3.1.tar.gz", hash = "sha256:ca53a428ac6a30984a894cf3847cf623b7dd7e8aadf8a63620b1f1f6155e4eaf", size = 361491, upload-time = "2026-08-21T23:06:13.993Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/1b/5b1964b0f1299aaca2b98300d73d972046ab70edb53c47674d6ace3c1bf1/dspy-3.4.0.tar.gz", hash = "sha256:4dea089c6bd1f8d314e1095e7f64b4b4ff64bb8a09549efee026c0a609f08c16" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/cc/d769426ff2cfb816bdedbcedf364c1812b6831887d34be8d188ca2727dc8/dspy-3.3.1-py3-none-any.whl", hash = "sha256:250049f565f52c014609ce2d3ca0de17a6c9449ac961492d61a009aa30dceabf", size = 421179, upload-time = "2026-08-21T23:06:12.591Z" }, + { url = "https://files.pythonhosted.org/packages/17/96/31628d4231b5dd7edbe9eaf5c975d1055f4b79b337062a1f957dca6dd931/dspy-3.4.0-py3-none-any.whl", hash = "sha256:36de329ecdaf18e31977e26c61b587caa470a34f574f2fe6cb7f4c82bcbf97cc" }, ] [[package]] @@ -751,7 +751,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "dspy", specifier = "==3.3.1" }, + { name = "dspy", specifier = "==3.4.0" }, { name = "httpx" }, { name = "numpy" }, { name = "pydantic" },