diff --git a/src/maple/core/calibration/shared_models.py b/src/maple/core/calibration/shared_models.py index 111b051..f729df7 100644 --- a/src/maple/core/calibration/shared_models.py +++ b/src/maple/core/calibration/shared_models.py @@ -367,6 +367,15 @@ class ObservedDistribution(BaseModel): "(across_patient / biological_experimental): it licenses the SD<->SEM round-trip " "and sets per-target finite-sample noise. Distinct from technical replicates.", ) + n_biological_is_floor: bool = Field( + default=False, + description="True when ``n_biological`` is a LOWER BOUND, not an exact count — the " + "source reports the unit count as 'n>=8', 'at least 8 donors', 'n=8-12 across " + "conditions', etc. Consumers that weight panels by precision (finite-sample noise " + "~ 1/sqrt(n), inverse-variance moment weighting) must treat a floor conservatively: " + "an exact-looking n from a floor over-states precision and over-weights the panel. " + "Leave False only when the source gives an exact per-summary n.", + ) n_technical: Optional[int] = Field( default=None, ge=1, diff --git a/src/maple/core/calibration/submodel_target.py b/src/maple/core/calibration/submodel_target.py index bedd328..8363772 100644 --- a/src/maple/core/calibration/submodel_target.py +++ b/src/maple/core/calibration/submodel_target.py @@ -23,6 +23,7 @@ from maple.core.calibration.code_validator import find_accessed_params from maple.core.calibration.exceptions import DimensionalityMismatchError from maple.core.calibration.shared_models import ( + DistributionShape, FigureExcerpt, ObservedDistribution, SourceRelevanceAssessment, @@ -1863,6 +1864,55 @@ def validate_observation_bootstrap_samples(self) -> "SubmodelTarget": return self + @model_validator(mode="after") + def validate_normal_error_stays_positive(self) -> "SubmodelTarget": + """Warn when an UN-clipped observation_code pushes a positive quantity negative. + + ``validate_clipping_suggests_lognormal`` catches additive-normal error models + that CLIP to avoid negatives. The complementary failure is an additive normal + that does NOT clip and simply emits unphysical negative samples for a + positive-only measurement — a constant additive SD over a low central value + (e.g. an MFI or count near the noise floor). Detected at runtime: a positive + median with a material negative tail and no clipping in the code. A genuinely + signed quantity (median ~ 0) is exempt; a clipped one is covered elsewhere. + """ + import numpy as np + + inputs_dict = {inp.name: inp.value for inp in self.inputs} + clip_patterns = ["np.clip", "np.maximum", "np.minimum", "max(0", "min("] + + for entry in self.calibration.error_model: + code = entry.observation_code + if not code or any(p in code for p in clip_patterns): + continue + try: + local_scope = {"np": np, "numpy": np} + exec(code, local_scope) + derive_observation = local_scope.get("derive_observation") + if derive_observation is None: + continue + sample_size = int(inputs_dict.get(entry.sample_size_input, 1)) + rng = np.random.default_rng(42) + result = derive_observation(inputs_dict, sample_size, rng, entry.n_bootstrap) + if not isinstance(result, np.ndarray) or result.size == 0: + continue + median = float(np.median(result)) + neg_frac = float(np.mean(result < 0)) + if median > 0 and neg_frac > 0.05: + warnings.warn( + f"Error model '{entry.name}': observation_code puts {neg_frac:.0%} of " + f"samples below zero for a positive-valued quantity (median={median:.3g}). " + "An additive normal with constant SD fits poorly near a low central value " + "— it emits unphysical negatives. Use a multiplicative/lognormal error " + "(rng.lognormal, or scale the width with the center) so samples stay " + "positive.", + UserWarning, + ) + except Exception: + pass # execution errors handled by other validators + + return self + @model_validator(mode="after") def validate_evaluation_points_within_span(self) -> "SubmodelTarget": """ @@ -2946,6 +2996,80 @@ def validate_clipping_suggests_lognormal(self) -> "SubmodelTarget": return self + @model_validator(mode="after") + def validate_center_channel_sem_scale(self) -> "SubmodelTarget": + """Enforce the two-channel contract when a population spread is declared. + + A target uses two channels: ``observation_code`` pins the CENTER (must be + SEM-scale, i.e. divide the per-draw width by ``sqrt(sample_size)``), and + ``observed_distribution`` carries the POPULATION spread (omega). SEM-scaling + is impossible without ``sample_size``, so an ``observation_code`` that never + references it is returning population-scale spread as the center likelihood — + encoding the spread TWICE (here AND in observed_distribution). This is the + prompt's "do not double-encode spread in both" rule made enforceable. Only + checked when the entry declares a population-spread ``observed_distribution``; + center-only / omitted-distribution entries are exempt. + """ + import ast + + for entry in self.calibration.error_model: + od = entry.observed_distribution + if od is None or not od.feeds_population_spread: + continue + code = entry.observation_code or "" + # 'sample_size' is always in the signature; detect BODY usage (a Name + # load), since SEM-scaling is impossible without referencing it. + try: + uses_sample_size = any( + isinstance(n, ast.Name) and n.id == "sample_size" + for n in ast.walk(ast.parse(code)) + ) + except SyntaxError: + continue # syntax errors handled by other validators + if not uses_sample_size: + raise ValueError( + f"Error model '{entry.name}' declares a population observed_distribution " + f"(spread_source='{od.spread_source.value}') but its observation_code never " + "uses 'sample_size' in its body. The observation_code is the CENTER channel " + "and must be SEM-scale: divide the per-draw width by sqrt(sample_size) so it " + "pins the mean, not the population. Without that, the population spread is " + "double-encoded — once here and once in observed_distribution (omega). " + "Return e.g. rng.normal(center, sd / np.sqrt(sample_size), n_bootstrap) and " + "let observed_distribution be the single source of population-spread truth." + ) + return self + + @model_validator(mode="after") + def validate_bounded_observable_uses_logit_normal(self) -> "SubmodelTarget": + """A bounded observable's population spread (``moments`` form) must use + ``shape: logit_normal``, not normal/lognormal. + + For a fraction / proportion / probability / percent observable, ``normal`` + puts mass outside the bound and ``lognormal`` is unbounded above (a near-1 + fraction's upper quartile escapes past 1). ``logit_normal`` expands the + quartiles in logit space so they stay in (0, 1). Only applies to the + ``moments`` form — the ``quantiles`` form carries the empirical shape (and + skew) directly and is exempt. + """ + BOUNDED_UNITS = {"percent", "%", "fraction", "proportion", "probability"} + for entry in self.calibration.error_model: + od = entry.observed_distribution + if od is None or od.moments is None: + continue + if od.moments.shape == DistributionShape.LOGIT_NORMAL: + continue + if (entry.units or "").strip().lower() not in BOUNDED_UNITS: + continue + raise ValueError( + f"Error model '{entry.name}' is a bounded observable (units='{entry.units}') " + f"but its observed_distribution.moments uses shape='{od.moments.shape.value}'. " + "Bounded fractions/percentages must use shape='logit_normal', which expands " + "quartiles in logit space so they never escape (0, 1); normal puts mass outside " + "the bound and lognormal is unbounded above. logit_normal requires center in " + "(0, 1) with center_type='median' — express a percent as a fraction (12% -> 0.12)." + ) + return self + @model_validator(mode="after") def validate_no_hardcoded_values_in_observation_code(self) -> "SubmodelTarget": """ diff --git a/src/maple/extraction/pipeline.py b/src/maple/extraction/pipeline.py index 7220376..30eb1cd 100644 --- a/src/maple/extraction/pipeline.py +++ b/src/maple/extraction/pipeline.py @@ -19,6 +19,7 @@ from pydantic import BaseModel, Field, model_validator from pydantic_ai import Agent, BinaryContent, WebSearchTool +from pydantic_ai.capabilities import NativeTool from pydantic_ai.models.openai import OpenAIResponsesModel, OpenAIResponsesModelSettings from pydantic_ai.providers.openai import OpenAIProvider @@ -1935,8 +1936,11 @@ def make_agents( "If your first searches find nothing, try synonyms, broader terms, or related biological concepts." ), model_settings=lit_search_settings, - # pydantic-ai >=2.x: native tools attach via capabilities= (was builtin_tools in 1.x) - capabilities=[WebSearchTool()], + # pydantic-ai >=2.x (post builtin->native rename, issue #5338): a native tool + # attaches via capabilities= wrapped in a NativeTool capability. Was + # builtin_tools=[WebSearchTool()] in 1.x; bare capabilities=[WebSearchTool()] + # fails at run time ("'WebSearchTool' object is not callable"). + capabilities=[NativeTool(WebSearchTool())], retries=max_retries, ) diff --git a/src/maple/prompts/submodel_target_prompt.md b/src/maple/prompts/submodel_target_prompt.md index d571be6..2d3cd9d 100644 --- a/src/maple/prompts/submodel_target_prompt.md +++ b/src/maple/prompts/submodel_target_prompt.md @@ -321,6 +321,10 @@ def derive_observation(inputs, sample_size, rng, n_bootstrap): **This width pins the CENTER, not the population.** The `observation_code` above divides by `sqrt(sample_size)`, so it returns an SEM-scale summary that constrains the parameter's *mean* — correct for this submodel's own inference. It is NOT the patient-to-patient population spread. When the paper reports genuine biological variability (across donors/animals/patients), declare it separately in the `observed_distribution` field of the error_model entry (next section) so hierarchical inference can use it as population spread. Without that, only the center is informed and the population spread falls back to a wide default. +**Do not double-encode the spread (VALIDATED).** If you declare a population `observed_distribution` on an entry, its `observation_code` MUST be SEM-scale — it must reference `sample_size` and divide the per-draw width by `sqrt(sample_size)`. Otherwise the population SD is encoded twice (once as the center width, once as omega) and the spread is double-counted. A validator rejects a population-spread entry whose `observation_code` never uses `sample_size` in its body. + +**Error-model SHAPE — use multiplicative/lognormal for positive, wide-range, or near-floor data.** An additive `rng.normal(center, sd, ...)` suits a signed or comfortably-positive quantity. For a strictly-positive readout (concentration, MFI, count, rate) — especially one spanning more than ~1 order of magnitude across doses, or sitting near a low noise floor — a constant additive SD emits unphysical NEGATIVE samples and a symmetric spread the data doesn't have. Use a **multiplicative/lognormal** error: either `rng.lognormal(mean=np.log(center), sigma=cv, size=n_bootstrap)`, or scale the width with the center (`scale = cv * center`). A validator warns when an un-clipped `observation_code` pushes a positive-median quantity below zero. (Reconstructing an SD from reported percentiles? Use the right z: a 5th–95th range spans `2 * 1.645` SDs, NOT `2 * 1.96` — 1.96 is for 2.5th–97.5th.) + --- ## Population Spread: `observed_distribution` (for hierarchical inference) @@ -369,10 +373,11 @@ observed_distribution: - Provide EXACTLY ONE of `moments` or `quantiles`. - A population spread (`biological_experimental` / `across_patient`) REQUIRES `n_biological` and `experimental_unit_type: biological`. A spread over technical/clonal replicates is not population variability — use `technical` or `center_only`. - `scale_type: sem` recovers the population SD as `SEM * sqrt(n_biological)` — so give `n_biological`. +- If the source reports the unit count as a LOWER BOUND ("n≥8", "at least 8 donors", "n=8–12 across conditions"), set `n_biological_is_floor: true` and use the floor value for `n_biological`. An exact-looking n from a floor over-states precision and over-weights the panel in the finite-sample / inverse-variance weighting downstream. - Keep values in paper units; do unit conversion in code, not in the anchors. - Most in-vitro submodel data is a donor/animal spread that is a LOWER BOUND on PDAC patient spread — grade that transfer in `source_relevance.heterogeneity_transfer` (see Source Relevance below). - `observation_code` still just pins the CENTER (keep it SEM-scale); `observed_distribution` is the single source of population-spread truth. Do not double-encode spread in both. -- For a `[0,1]`-bounded observable (a fraction, proportion, or probability — e.g. a polarization fraction, a positive-cell %), use `shape: logit_normal` instead of `lognormal`. It expands the quartiles in logit space so they can never escape `(0, 1)`; `lognormal` on a near-1 fraction would push the upper quartile past 1. `logit_normal` requires `center_type: median`. +- For a `[0,1]`-bounded observable (a fraction, proportion, or probability — e.g. a polarization fraction, a positive-cell %), use `shape: logit_normal` instead of `lognormal` in the `moments` form. It expands the quartiles in logit space so they can never escape `(0, 1)`; `lognormal` on a near-1 fraction would push the upper quartile past 1. `logit_normal` requires `center_type: median`. **This includes `percent` observables** — express the value as a fraction in `(0, 1)` first (12% → 0.12), since `logit_normal` needs the center inside the unit interval. A validator rejects a `percent`/`fraction`/`proportion`/`probability` observable whose `moments` shape is `normal` or `lognormal`. (The `quantiles` form is exempt — explicit anchors carry the empirical shape and skew directly, and are often the better choice for a visibly-skewed donor distribution.) **`unit_group` — only for multi-observable, SAME-unit targets.** When a single target has SEVERAL `error_model` entries measured on the SAME biological units (the same donor panel across the doses of a dose-response; one cohort followed over a time course), tag those entries with a shared `unit_group` string. This tells the hierarchical layer they share ONE biological random effect, so it moment-matches them jointly instead of treating each dose/timepoint as an independent measurement (which would spuriously shrink the population spread by ~sqrt(number-of-points)). diff --git a/tests/unit/core/test_submodel_target_validators.py b/tests/unit/core/test_submodel_target_validators.py index 1df3380..8230ded 100644 --- a/tests/unit/core/test_submodel_target_validators.py +++ b/tests/unit/core/test_submodel_target_validators.py @@ -1960,3 +1960,153 @@ def test_multiple_source_inputs(self): } ) SubmodelTarget(**data) + + +# ============================================================================ +# Tests for validate_center_channel_sem_scale (R1) — two-channel "do not +# double-encode": when a population observed_distribution is declared, the +# observation_code (center channel) MUST be SEM-scale (use sample_size). +# ============================================================================ + +# center channel that correctly SEM-scales (divides by sqrt(sample_size)) +SEM_SCALE_OBSERVATION_CODE = """ +def derive_observation(inputs, sample_size, rng, n_bootstrap): + import numpy as np + return rng.normal(inputs['test_value'], 1.0 / np.sqrt(sample_size), n_bootstrap) +""" + +# center channel that does NOT use sample_size — population-scale, double-encodes +NON_SEM_OBSERVATION_CODE = """ +def derive_observation(inputs, sample_size, rng, n_bootstrap): + import numpy as np + return rng.normal(inputs['test_value'], 1.0, n_bootstrap) +""" + + +def _population_observed_distribution() -> dict: + return { + "quantiles": [ + {"p": 0.25, "value": 8.0}, + {"p": 0.5, "value": 10.0}, + {"p": 0.75, "value": 13.0}, + ], + "spread_source": "biological_experimental", + "n_biological": 10, + "experimental_unit_type": "biological", + } + + +class TestCenterChannelSemScale: + """R1: population observed_distribution requires an SEM-scale observation_code.""" + + def test_population_spread_without_sample_size_raises(self): + data = make_algebraic_target( + input_value=10.0, measurement_error_code=NON_SEM_OBSERVATION_CODE + ) + data["calibration"]["error_model"][0][ + "observed_distribution" + ] = _population_observed_distribution() + with pytest.raises(ValidationError, match="sample_size"): + SubmodelTarget(**data) + + def test_population_spread_with_sem_scale_passes(self): + data = make_algebraic_target( + input_value=10.0, measurement_error_code=SEM_SCALE_OBSERVATION_CODE + ) + data["calibration"]["error_model"][0][ + "observed_distribution" + ] = _population_observed_distribution() + # R1 must not fire; construction succeeds. + SubmodelTarget(**data) + + def test_center_only_without_sample_size_is_exempt(self): + # No population spread => R1 does not apply even without sample_size. + data = make_algebraic_target( + input_value=10.0, measurement_error_code=NON_SEM_OBSERVATION_CODE + ) + SubmodelTarget(**data) + + +# ============================================================================ +# Tests for validate_normal_error_stays_positive (V-A) — un-clipped additive +# normal that pushes a positive quantity below zero. +# ============================================================================ + +# additive normal, scale ~= center => a fat negative tail, no clipping +CROSSES_ZERO_OBSERVATION_CODE = """ +def derive_observation(inputs, sample_size, rng, n_bootstrap): + import numpy as np + return rng.normal(inputs['test_value'], inputs['test_value'], n_bootstrap) +""" + + +class TestNormalErrorStaysPositive: + """V-A: warn when an un-clipped normal emits negatives for a positive quantity.""" + + def test_normal_crossing_zero_warns(self): + data = make_algebraic_target( + input_value=5.0, measurement_error_code=CROSSES_ZERO_OBSERVATION_CODE + ) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + try: + SubmodelTarget(**data) + except ValidationError: + pass + hits = [x for x in w if "below zero" in str(x.message)] + assert len(hits) > 0 + + def test_well_behaved_positive_normal_no_warning(self): + # scale small relative to center => negligible negative mass, no warning. + data = make_algebraic_target(input_value=100.0) # default scale=1.0 + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + try: + SubmodelTarget(**data) + except ValidationError: + pass + hits = [x for x in w if "below zero" in str(x.message)] + assert len(hits) == 0 + + +# ============================================================================ +# Tests for validate_bounded_observable_uses_logit_normal (V-B) — bounded +# observable in the moments form must use shape='logit_normal'. +# ============================================================================ + + +def _percent_target(shape: str) -> dict: + """A percent-unit target with a moments-form population spread of the given shape.""" + data = make_algebraic_target( + input_value=0.12, measurement_error_code=SEM_SCALE_OBSERVATION_CODE + ) + # Make units internally consistent as a bounded fraction (percent). + data["inputs"][0]["units"] = "percent" + data["calibration"]["parameters"][0]["units"] = "percent" + em = data["calibration"]["error_model"][0] + em["units"] = "percent" + em["observed_distribution"] = { + "moments": { + "center": 0.12, + "center_type": "median", + "scale": 0.05, + "scale_type": "sd", + "shape": shape, + }, + "spread_source": "biological_experimental", + "n_biological": 10, + "experimental_unit_type": "biological", + } + return data + + +class TestBoundedObservableLogitNormal: + """V-B: bounded moments-form observable must use logit_normal.""" + + def test_percent_with_normal_shape_raises(self): + with pytest.raises(ValidationError, match="logit_normal"): + SubmodelTarget(**_percent_target("normal")) + + def test_percent_with_logit_normal_passes(self): + # logit_normal on a median-in-(0,1) percent observable is the correct shape. + SubmodelTarget(**_percent_target("logit_normal"))