From d596ea4264fe413be7e8d800d8faf1a91dff076a Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Thu, 17 Sep 2026 19:40:24 -0700 Subject: [PATCH 01/15] Implement experiment metrics and reported custom measurements --- .dockerignore | 1 - .env.example | 5 + .gitignore | 4 + CLAUDE.md | 14 + CONTEXT.md | 106 ++ Dockerfile | 24 +- README.md | 43 + compose.yml | 26 + ...01-measurements-have-explicit-producers.md | 17 + frontend/package-lock.json | 986 +++++++++++++++++- frontend/package.json | 8 +- frontend/src/api/client.ts | 41 +- .../CreateExperimentFromFileDialog.tsx | 9 +- .../src/components/protocol/AddNodePanel.tsx | 2 +- .../protocol/AgentNodeInspector.tsx | 43 - .../protocol/CreateCustomMetricDialog.tsx | 116 +++ .../protocol/CustomMetricDialogs.test.tsx | 136 +++ .../protocol/CustomMetricDialogs.tsx | 268 +++++ .../src/components/protocol/DesignTab.tsx | 332 ++---- .../protocol/FactorBindableField.tsx | 11 +- .../protocol/FactorEditorDialog.tsx | 135 ++- .../protocol/McpToolNodeInspector.tsx | 1 + .../src/components/protocol/MetricPanel.tsx | 62 ++ .../protocol/MetricsEditor.autosave.test.tsx | 97 ++ .../MetricsEditor.availability.test.tsx | 81 ++ .../src/components/protocol/MetricsEditor.tsx | 511 +++++++++ .../protocol/NodeInspectorDialog.tsx | 4 +- .../protocol/ProtocolCanvas.test.tsx | 208 ++++ .../components/protocol/ProtocolCanvas.tsx | 186 +++- .../protocol/ProtocolCanvasContext.tsx | 5 + .../protocol/ProtocolCanvasMenu.tsx | 1 + .../components/protocol/ResultsTab.test.tsx | 277 +++++ .../src/components/protocol/ResultsTab.tsx | 187 +++- .../components/protocol/RunConfirmDialog.tsx | 6 +- frontend/src/components/protocol/RunsTab.tsx | 15 +- .../protocol/ScriptNodeInspector.tsx | 1 + .../protocol/TestRunResults.test.tsx | 177 ++++ .../components/protocol/TestRunResults.tsx | 131 +++ .../src/components/protocol/bindableFields.ts | 2 +- .../components/protocol/nodes/AgentNode.tsx | 49 +- .../components/protocol/nodes/CircleNode.tsx | 5 + .../protocol/nodes/NodeMetricBadge.tsx | 19 + .../components/protocol/runSummary.test.ts | 28 + .../src/components/protocol/runSummary.ts | 7 +- .../protocol/useDialogAutosave.test.tsx | 70 ++ .../components/protocol/useDialogAutosave.ts | 105 ++ frontend/src/lib/agentOutputMetrics.ts | 63 ++ .../src/lib/contextualMetricSuggestions.ts | 69 ++ frontend/src/lib/customMetrics.ts | 43 + frontend/src/lib/experiment.ts | 2 +- frontend/src/lib/mcpToolMetrics.ts | 46 + frontend/src/lib/measurementPlan.ts | 169 +++ frontend/src/lib/metricCatalog.ts | 104 +- frontend/src/lib/pythonScriptMetrics.ts | 79 ++ frontend/src/lib/reportedMetrics.test.ts | 54 + frontend/src/pages/ProtocolCanvasPage.tsx | 2 +- frontend/src/test/setup.ts | 5 + frontend/src/types/experiments.ts | 133 ++- frontend/src/types/mcpServers.ts | 8 + frontend/src/types/protocols.ts | 46 +- frontend/vitest.config.ts | 17 + publications/bioinformatics/README.md | 13 +- pyproject.toml | 2 +- scripts/promote_score_metrics.py | 53 - sdk/src/asaree_client/models.py | 2 + .../asaree_client/resources/experiments.py | 6 + src/asaree/api/experiments.py | 345 ++++-- src/asaree/api/mcp_servers.py | 59 +- src/asaree/api/protocols.py | 174 +++- src/asaree/config.py | 7 + src/asaree/mcp_servers/script_server.py | 125 ++- ...9e4a7b2c1d30_add_metric_recommendations.py | 30 + .../c6d7e8f9a0b1_add_measurement_plan.py | 37 + ...7e8f9a0b1c2_add_protocol_run_wall_clock.py | 32 + .../f1a2b3c4d5e6_add_latest_test_run.py | 42 + src/asaree/models/experiment.py | 14 + src/asaree/models/protocol_run.py | 18 +- src/asaree/services/agent_messenger.py | 8 +- src/asaree/services/csv_export.py | 140 ++- .../services/experiment_measurements.py | 212 ++++ src/asaree/services/experiment_run_results.py | 147 ++- src/asaree/services/experiments.py | 34 + src/asaree/services/measurement_engine.py | 967 +++++++++++++++++ src/asaree/services/measurement_migration.py | 143 +++ src/asaree/services/metric_evaluation.py | 83 -- src/asaree/services/metric_promotion.py | 260 ----- src/asaree/services/metrics.py | 319 +++--- src/asaree/services/protocol_execution.py | 602 ++++------- src/asaree/services/protocol_graph.py | 37 + src/asaree/services/protocol_runs.py | 246 ++++- src/asaree/services/reported_metrics.py | 421 ++++++++ src/asaree/services/runtime_metrics.py | 689 ++++++++++++ src/asaree/services/test_run_results.py | 170 +++ src/asaree/worker/enqueue.py | 8 - src/asaree/worker/settings.py | 2 - src/asaree/worker/tasks.py | 10 +- tests/test_agent_messenger.py | 4 +- tests/test_csv_export.py | 34 + tests/test_experiment_measurements.py | 698 +++++++++++++ tests/test_experiment_run_results.py | 156 ++- tests/test_experiments.py | 446 ++++++++ tests/test_mcp_server_annotations.py | 32 + tests/test_measurement_engine.py | 845 +++++++++++++++ tests/test_measurement_migration.py | 111 ++ tests/test_metric_promotion.py | 179 ---- tests/test_metrics.py | 260 +++-- tests/test_protocol_execution.py | 182 ++-- tests/test_protocol_runs.py | 514 ++++++++- tests/test_reported_metrics.py | 237 +++++ tests/test_runtime_metrics.py | 417 ++++++++ tests/test_script_server.py | 38 +- uv.lock | 6 +- 112 files changed, 12893 insertions(+), 2125 deletions(-) create mode 100644 CONTEXT.md create mode 100644 docs/adr/0001-measurements-have-explicit-producers.md create mode 100644 frontend/src/components/protocol/CreateCustomMetricDialog.tsx create mode 100644 frontend/src/components/protocol/CustomMetricDialogs.test.tsx create mode 100644 frontend/src/components/protocol/CustomMetricDialogs.tsx create mode 100644 frontend/src/components/protocol/MetricPanel.tsx create mode 100644 frontend/src/components/protocol/MetricsEditor.autosave.test.tsx create mode 100644 frontend/src/components/protocol/MetricsEditor.availability.test.tsx create mode 100644 frontend/src/components/protocol/MetricsEditor.tsx create mode 100644 frontend/src/components/protocol/ProtocolCanvas.test.tsx create mode 100644 frontend/src/components/protocol/ResultsTab.test.tsx create mode 100644 frontend/src/components/protocol/TestRunResults.test.tsx create mode 100644 frontend/src/components/protocol/TestRunResults.tsx create mode 100644 frontend/src/components/protocol/nodes/NodeMetricBadge.tsx create mode 100644 frontend/src/components/protocol/runSummary.test.ts create mode 100644 frontend/src/components/protocol/useDialogAutosave.test.tsx create mode 100644 frontend/src/components/protocol/useDialogAutosave.ts create mode 100644 frontend/src/lib/agentOutputMetrics.ts create mode 100644 frontend/src/lib/contextualMetricSuggestions.ts create mode 100644 frontend/src/lib/customMetrics.ts create mode 100644 frontend/src/lib/mcpToolMetrics.ts create mode 100644 frontend/src/lib/measurementPlan.ts create mode 100644 frontend/src/lib/pythonScriptMetrics.ts create mode 100644 frontend/src/lib/reportedMetrics.test.ts create mode 100644 frontend/src/test/setup.ts create mode 100644 frontend/vitest.config.ts delete mode 100644 scripts/promote_score_metrics.py create mode 100644 src/asaree/migrations/versions/9e4a7b2c1d30_add_metric_recommendations.py create mode 100644 src/asaree/migrations/versions/c6d7e8f9a0b1_add_measurement_plan.py create mode 100644 src/asaree/migrations/versions/d7e8f9a0b1c2_add_protocol_run_wall_clock.py create mode 100644 src/asaree/migrations/versions/f1a2b3c4d5e6_add_latest_test_run.py create mode 100644 src/asaree/services/experiment_measurements.py create mode 100644 src/asaree/services/measurement_engine.py create mode 100644 src/asaree/services/measurement_migration.py delete mode 100644 src/asaree/services/metric_evaluation.py delete mode 100644 src/asaree/services/metric_promotion.py create mode 100644 src/asaree/services/protocol_graph.py create mode 100644 src/asaree/services/reported_metrics.py create mode 100644 src/asaree/services/runtime_metrics.py create mode 100644 src/asaree/services/test_run_results.py create mode 100644 tests/test_experiment_measurements.py create mode 100644 tests/test_mcp_server_annotations.py create mode 100644 tests/test_measurement_engine.py create mode 100644 tests/test_measurement_migration.py delete mode 100644 tests/test_metric_promotion.py create mode 100644 tests/test_reported_metrics.py create mode 100644 tests/test_runtime_metrics.py diff --git a/.dockerignore b/.dockerignore index e693c25..ce2491d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,6 +19,5 @@ data/ # committed and later removed from it, into an image layer. .git/ sdk/ -tests/ *.log *_out.ipynb diff --git a/.env.example b/.env.example index 77b2255..f88cb52 100644 --- a/.env.example +++ b/.env.example @@ -137,3 +137,8 @@ ASAREE_OKF_BUNDLE_ROOT=~ # not in core: core has no way to know how slow any given product's tools or # provider responses are. ASAREE_HOOK_TIMEOUT_SECONDS=3600 + +# Administrative limits for independent post-run metric producers. These are +# server policy, not per-metric controls exposed in measurement plans. +ASAREE_METRIC_PRODUCER_MAX_CONCURRENCY=4 +ASAREE_METRIC_PRODUCER_TIMEOUT_SECONDS=30 diff --git a/.gitignore b/.gitignore index b79d76c..558ed26 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ backups/ # Generated by hatch-vcs at build time from the git tag (see pyproject.toml). src/asaree/_version.py sdk/src/asaree_client/_version.py + +# Private local planning and agent guidance +docs/ +issues/ diff --git a/CLAUDE.md b/CLAUDE.md index 7c477a2..73f237f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,20 @@ or introduce a different aesthetic without being asked. leaving the canvas ≥420px) and remembers its width in `localStorage` — so a new tab in it should be width-responsive via container queries, not built for one fixed column width. +## Agent skills + +### Issue tracker + +Implementation issues and specs live in the separate `EpistasisLab/ASAREE_Issues` GitHub repository. See `docs/agents/issue-tracker.md`. + +### Triage labels + +Use the five canonical Matt Pocock triage labels. See `docs/agents/triage-labels.md`. + +### Domain docs + +This is a single-context repository with `CONTEXT.md` at the root and private local ADRs under `docs/adr/`. See `docs/agents/domain.md`. + # Git commit conventions Do not add `Co-Authored-By` or `Generated-with` lines to commits or PRs. diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..8d1d770 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,106 @@ +# ASAREE + +ASAREE runs factorial experiments over agent protocols: a user declares factors, +metrics, and an agent protocol graph on a canvas; the harness executes every +factor-combination cell and compares the recorded outcomes. + +## Language + +### Experiments + +**Experiment**: +A factorial study comparing agent protocol conditions. Owns the design declaration, the protocol canvas, and all runs. +_Avoid_: Project, study, trial + +**Factor**: +A declared experimental variable with discrete levels, bound to at least one canvas field before cells can run. +_Avoid_: Parameter, variable + +**Cell**: +One unique factor combination together with all its planned replicates. +_Avoid_: Condition, treatment (accepted in prose, never in code) + +**Replicate**: +One run of one cell. Counts of pending/run/scored are replicate counts, never cell counts. + +**Primary metric**: +The optional declared metric Results uses to rank cells. An editable experiment may have none; when declared, it must be unique. +_Avoid_: Main metric, target metric + +**Design revision**: +An immutable snapshot of the generated design. The current design is the one revision with `superseded_at IS NULL`. +_Avoid_: Design version + +### Metrics + +**Measurement plan**: +The experiment's declared set of metrics, their producers, and the bindings that provide each producer's inputs. +_Avoid_: Metric list, evaluation config + +**Metric**: +A scalar experimental outcome that can be observed per replicate and aggregated across a cell. +_Avoid_: Measurement, score, result field + +**Built-in metric**: +A metric whose definition and producer ASAREE supplies as one supported capability. +_Avoid_: Catalog metric, preset metric + +**Custom metric**: +An experiment-owned metric whose definition or producer configuration is supplied by the user. +_Avoid_: User metric, freeform metric + +**Metric producer**: +The declared mechanism that observes a metric from a completed replicate. A producer may emit several metrics and evaluation artifacts from one evaluation. +_Avoid_: Calculator, metric tool, extraction rule + +**Runtime producer**: +A metric producer that observes execution facts recorded by the harness, such as cost, tokens, duration, or tool calls. +_Avoid_: Telemetry metric, process metric + +**Deterministic evaluator**: +A metric producer that applies a reproducible evaluation procedure to declared run outputs and reference inputs. +_Avoid_: Score agent, metric script + +**Model judge**: +A metric producer that applies a declared rubric to run output using a selected language model. +_Avoid_: Critic, evaluator agent + +**Reported metric**: +A display/export-only custom metric containing either a configured Agent's completed +final output or the opaque result of its last matching Script or MCP tool call. ASAREE +never invokes the source to produce it, validates no result type, and leaves the value +absent when the Agent did not complete or call the configured tool. +_Avoid_: Agent metric, parsed metric + +**Metric observation**: +One producer's outcome for one metric on one replicate, including whether it was measured, unavailable, failed, timed out, cancelled, or not applicable. +_Avoid_: Metric value, score record + +**Evaluation artifact**: +A structured diagnostic such as a confusion matrix, calibration curve, or per-class report that accompanies observations but is not ranked or aggregated as a metric. +_Avoid_: Array metric, report metric + +**Legacy value**: +A historical raw value outside the canonical aggregatable metric contract whose original producer was not recorded. It stays visible with `legacy.unknown` provenance but is not ranked; ASAREE never infers its producer from a display name, prompt, or tool call. +_Avoid_: Inferred observation, promoted score + +**Context metric**: +A metric an Agent inspector includes in that agent's system prompt as evaluation guidance; it never exposes a future observation. +_Avoid_: Prompt metric + +### Runs + +**Protocol canvas**: +The draft editable graph. Production runs never read it directly. +_Avoid_: Workflow, pipeline + +**Published revision**: +The immutable protocol snapshot a run executes; created by publishing the canvas. +_Avoid_: Protocol version + +**Gated pair**: +An agent node with its critic gate. `approved` and `revisions_used` are recorded on the worker's node-run. +_Avoid_: Reviewer, supervisor + +**Attempt**: +One ProtocolRun for a replicate; later attempts supersede earlier ones but every attempt's own facts stay immutable. diff --git a/Dockerfile b/Dockerfile index e94f4bf..55a6fa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # python -m ...`, the same subprocess convention used in every dev environment # so far. Removing uv here would just move the same "no uv at runtime" problem # one level down. -FROM python:3.13-slim +FROM python:3.13-slim AS application WORKDIR /app @@ -111,3 +111,25 @@ ENV PATH="/app/.venv/bin:$PATH" EXPOSE 8000 CMD ["uvicorn", "asaree.app:app", "--host", "0.0.0.0", "--port", "8000"] + +# Opt-in test image used by the asaree-tests Compose service. Keep pytest and +# the test tree out of the production image while reusing every application +# dependency layer above. The git context is required for the same hatch-vcs +# reason as the application install. +FROM application AS test + +COPY tests/ ./tests/ +# One backend contract test compares the mirrored TypeScript catalog with the +# Python source of truth. Keep the test image narrow while making that source +# available at the same repository-relative path used outside containers. +COPY frontend/src/lib/metricCatalog.ts ./frontend/src/lib/metricCatalog.ts +RUN --mount=type=bind,from=gitdir,target=/app/.git \ + --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + uv sync --frozen --group dev + +CMD ["pytest", "tests/", "-q", "--tb=short"] + +# Keep the default build target as the production application. Merely adding +# the test stage must not put test-only dependencies or source in deployed +# images built without an explicit target. +FROM application AS runtime diff --git a/README.md b/README.md index 58cade8..a249b07 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # ASAREE +[![Latest release](https://img.shields.io/github/v/release/EpistasisLab/ASAREE?display_name=tag&sort=semver)](https://github.com/EpistasisLab/ASAREE/releases/latest) +[![CI](https://github.com/EpistasisLab/ASAREE/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/EpistasisLab/ASAREE/actions/workflows/ci.yml) +[![Python](https://img.shields.io/badge/Python-%E2%89%A53.12-3776AB?logo=python&logoColor=white)](https://github.com/EpistasisLab/ASAREE/blob/main/pyproject.toml) +[![License](https://img.shields.io/github/license/EpistasisLab/ASAREE)](https://github.com/EpistasisLab/ASAREE/blob/main/LICENSE) + **A**nalytical **S**andbox for **A**gentic **R**esearch, **E**ngineering, and **E**xperimentation — a workbench for running LLM agents as designed experiments rather than one-off prompts. @@ -86,6 +91,31 @@ docker compose up -d # picks up the changed .env — five agents in series building a classifier on a public dataset. It picks up exactly where this step leaves off. +## Measurements and Results + +The Design panel offers built-in runtime and supervised-ML metrics with their +producer, required inputs, units, aggregation, direction, and current readiness. +Custom metrics can read a typed numeric/Boolean structured-output field or use a +model judge with an explicit model, rubric, bounds, and selected run inputs. A +judge is an extra model call on every replicate, so its recurring call and cost +estimate is shown before save. The optional wizard creates the same typed draft +as the manual editor and cannot bypass the normal validator. + +Every completed attempt records one state for every declared metric: +`measured`, `unavailable` (the required value was absent), `failed` (the producer +could not evaluate it), or `not_applicable` (the metric does not apply to that +task). Results keeps those states distinct. Confusion matrices, calibration +data, and per-class reports are evaluation artifacts: they appear in attempt +detail but are never ranked or averaged as scalar metrics. + +The Results CSV exports scalar values as ordinary analysis columns. Its +`observation_statuses` JSON column retains each metric's state and error, and +`evaluation_artifacts` retains structured diagnostics without flattening them +into misleading scalar columns. The `legacy_values` JSON column keeps old text +or other non-rankable values with `legacy.unknown` provenance when their +original producer was never recorded; choose an explicit producer before +running that legacy declaration again. + ## Everyday commands ```bash @@ -95,6 +125,19 @@ docker compose restart asaree-app # apply an edited .env docker compose down # stop, keep all data ``` +Run the backend test suite in Compose when you do not already have the dev +database exposed to the host. The one-shot runner waits for both migration +chains and connects to Postgres over the internal Compose network, so it does +not depend on `localhost`, `POSTGRES_PORT`, or a host PostgreSQL installation: + +```bash +docker compose run --rm --build asaree-tests +``` + +The tests themselves still run under pytest; only pytest and its real Postgres +dependency are placed on the same network. The `test` profile keeps this +service out of a normal `docker compose up`. + The frontend hot-reloads from your checkout; backend changes need a rebuild. `docker compose up -d --build` also runs pending database migrations. The diff --git a/compose.yml b/compose.yml index ccb3b49..ec087de 100644 --- a/compose.yml +++ b/compose.yml @@ -218,6 +218,32 @@ services: - ./data:/app/data restart: unless-stopped + # Integration-test runner on the Compose network. The tests intentionally + # use real Postgres (JSONB and core's vector schema rule out SQLite), so run + # them beside the database instead of relying on a host-side localhost port. + # The profile keeps this one-shot service out of ordinary `docker compose + # up`; targeting it explicitly activates it: + # + # docker compose run --rm --build asaree-tests + asaree-tests: + profiles: ["test"] + build: + context: . + target: test + secrets: + - gh_token + additional_contexts: + gitdir: ./.git + depends_on: + asaree-migrate: + condition: service_completed_successfully + environment: + DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro + ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro + ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/asaree + ASAREE_REDIS_URL: redis://motoro-redis:6379/0 + restart: "no" + asaree-frontend: build: context: ./frontend diff --git a/docs/adr/0001-measurements-have-explicit-producers.md b/docs/adr/0001-measurements-have-explicit-producers.md new file mode 100644 index 0000000..2676112 --- /dev/null +++ b/docs/adr/0001-measurements-have-explicit-producers.md @@ -0,0 +1,17 @@ +# Built-in metrics are evaluated; custom metrics are reported + +ASAREE distinguishes two measurement paths. + +Built-in metrics are owned by ASAREE. Their runtime producer reads the immutable attempt snapshot during finalization, computes supported numeric or Boolean observations, and supplies the values used for aggregation, comparison, and ranking. + +Custom metrics are owned by the experiment author. A custom declaration points to an Agent, or to a Script/MCP tool directly connected to that Agent. ASAREE does not execute that source as a post-run evaluator. It captures the Agent's completed final output or the result of the last matching tool call already recorded in the Agent's execution trace. The last call is used whether it succeeded or failed. + +## Consequences + +- Users are responsible for prompts, scripts, MCP tools, and Agent behavior that produce the value they want to record. +- Custom values are opaque JSON. ASAREE does not coerce, validate, aggregate, rank, or choose a primary custom metric. +- If the Agent never produces the configured output or call, the observation is `unavailable` and its CSV cell is empty. +- Test Runs, per-node Play, production Results, and CSV export all consume the same persisted observation shape. +- Every observation retains producer provenance and a status. A measured JSON `null` remains distinguishable from an unavailable observation through that status and is exported as the literal `null`. +- Script and MCP configuration is validated for source identity, enabled state, and direct Agent wiring. Evaluator input mappings, result paths, deterministic execution, and side-effect acknowledgements are not part of custom measurement. +- Historical `deterministic_evaluator` bindings are normalized at the persistence boundary to reported bindings; no runtime adapter exists for them. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a8f3b48..b128c94 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -27,16 +27,74 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.3.3", + "@testing-library/jest-dom": "^7.0.1", + "@testing-library/react": "^16.3.3", + "@testing-library/user-event": "^14.6.7", "@types/node": "^24.13.3", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.4", + "jsdom": "^29.0.0", "oxlint": "^1.75.0", "tailwindcss": "^4.3.3", "typescript": "~6.0.2", - "vite": "^8.2.0" + "vite": "^8.2.0", + "vitest": "^5.0.1" } }, + "node_modules/@adobe/css-tools": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.5.0.tgz", + "integrity": "sha512-6OzddxPio9UiWTCemp4N8cYLV2ZN1ncRnV1cVGtve7dhPOtRkleRyx32GQCYSwDYgaHU3USMm84tNsvKzRCa1Q==", + "dev": true + }, + "node_modules/@asamuzakjp/css-color": { + "version": "5.1.11", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", + "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", + "dev": true, + "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", + "@csstools/css-calc": "^3.2.0", + "@csstools/css-color-parser": "^4.1.0", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/dom-selector": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz", + "integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==", + "dev": true, + "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", + "@asamuzakjp/nwsapi": "^2.3.9", + "bidi-js": "^1.0.3", + "css-tree": "^3.2.1", + "is-potential-custom-element-name": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/generational-cache": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", + "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", + "dev": true, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/nwsapi": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", + "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", + "dev": true + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -480,6 +538,18 @@ } } }, + "node_modules/@bramus/specificity": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", + "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", + "dev": true, + "dependencies": { + "css-tree": "^3.0.0" + }, + "bin": { + "specificity": "bin/cli.js" + } + }, "node_modules/@codemirror/autocomplete": { "version": "6.20.3", "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.3.tgz", @@ -577,6 +647,140 @@ "w3c-keyname": "^2.2.4" } }, + "node_modules/@csstools/color-helpers": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.1.tgz", + "integrity": "sha512-gLNsunvwf3mCi5u5o46/Z/JcJMnhbHSaZ69rkgPzNM3J4s8hWwpPUQB6/tt0EDFyCiWzxANlx+2LJwpYj4zS1w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@csstools/css-calc": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.0.tgz", + "integrity": "sha512-XQKj5B7QiZcHiegCOCAzcAOJdhGgWOHbbu62h5e5mkHnn8lWcfiJhllkqWmxu5zWR9jucPHuo1iTB56P033hcg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.3.tgz", + "integrity": "sha512-y4LpL+lmpuyKDiEFq2PnZUVFdAjsoB/qQJod79yLNokXyW7jewi+/WJ69EfItj8A2unWtxXnGjw6LYXgXu5ZjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "dependencies": { + "@csstools/color-helpers": "^6.1.1", + "@csstools/css-calc": "^3.4.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", + "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.14.tgz", + "integrity": "sha512-HpbVXyrofRXpHpgkNIjU/3EWR4WJvOkO3emNK/L6X/mTJU7bGUI3AkkpoTNXznQLp0KRjLHELTGeKI5dIkI9JQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "peerDependencies": { + "css-tree": "^3.2.1" + }, + "peerDependenciesMeta": { + "css-tree": { + "optional": true + } + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", + "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/@dotenvx/dotenvx": { "version": "1.75.1", "resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-1.75.1.tgz", @@ -744,6 +948,23 @@ "resolved": "https://registry.npmjs.org/@dotenvx/primitives/-/primitives-0.8.0.tgz", "integrity": "sha512-VYJy0uhFm9zTJ1TxBaW/pA8bjbOM/OttaNMwZ1RHG4JKyRG7DhSdiqD1ipQoAyoD22olUtxbP78W9xY3Wd11bg==" }, + "node_modules/@exodus/bytes": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz", + "integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==", + "dev": true, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "@noble/hashes": "^1.8.0 || ^2.0.0" + }, + "peerDependenciesMeta": { + "@noble/hashes": { + "optional": true + } + } + }, "node_modules/@floating-ui/core": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", @@ -824,9 +1045,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==" + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", @@ -2033,6 +2254,100 @@ "react": "^18 || ^19" } }, + "node_modules/@testing-library/dom": { + "version": "10.4.2", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.2.tgz", + "integrity": "sha512-yzr2S9HyAIdhz2/6qHgbs665Q7PKVcDF05vsOlHPxG1mo36gKVesdYVeDLnXgfjJ03CrKRk08knc6+E/9m8v2Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@testing-library/jest-dom": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-7.0.1.tgz", + "integrity": "sha512-oMDTC3oA+6CXSO2JZnvOI7CA6oVub6kij5ggk9ohwye5slmkwxYDXcPOVxgMw/RQlticjtO0C1RZkR97HgrWMw==", + "dev": true, + "dependencies": { + "@adobe/css-tools": "^4.4.0", + "aria-query": "^5.0.0", + "css.escape": "^1.5.1", + "dom-accessibility-api": "^0.6.3", + "picocolors": "^1.1.1", + "redent": "^3.0.0" + }, + "engines": { + "node": ">=22", + "npm": ">=6", + "yarn": ">=1" + }, + "peerDependencies": { + "@testing-library/dom": ">=10 <11", + "vitest": ">= 0.32" + }, + "peerDependenciesMeta": { + "vitest": { + "optional": true + } + } + }, + "node_modules/@testing-library/jest-dom/node_modules/dom-accessibility-api": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz", + "integrity": "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==", + "dev": true + }, + "node_modules/@testing-library/react": { + "version": "16.3.3", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.3.tgz", + "integrity": "sha512-Uo193NgQbPMz6lrrhtRQQFcMC6Re/ELLFbbuVL30WDlZxlpZf9/lMHTAVxPRLw1q1iu9OJmR1c2BLiENRstdBg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.12.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@testing-library/dom": "^10.0.0", + "@types/react": "^18.0.0 || ^19.0.0", + "@types/react-dom": "^18.0.0 || ^19.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@testing-library/user-event": { + "version": "14.6.7", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.7.tgz", + "integrity": "sha512-MPCpX8bxe8zS+JmmTwLp8jd0dy1rAm60Te/SL8JrQM3qvQJcBOs1d7IefJMyZzqM3EWBrDn/LWDt1BCGu4ASfg==", + "dev": true, + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@testing-library/dom": ">=7.21.4" + } + }, "node_modules/@ts-morph/common": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.27.0.tgz", @@ -2043,6 +2358,23 @@ "path-browserify": "^1.0.1" } }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "peer": true + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/d3-color": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", @@ -2086,6 +2418,18 @@ "@types/d3-selection": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true + }, "node_modules/@types/node": { "version": "24.13.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", @@ -2194,6 +2538,51 @@ } } }, + "node_modules/@vitest/mocker": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.1.tgz", + "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", + "@vitest/spy": "5.0.1", + "estree-walker": "^3.0.3", + "magic-string": "^1.2.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/mocker/node_modules/magic-string": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.1.tgz", + "integrity": "sha512-8lyCu36ErXR0J9uaGKlKQoiLZKmtI63YGLE8G2o9jyRPdr4X47LusSOwgOJOzcVtp81fTAAjxR7BwKz682Jhow==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, + "node_modules/@vitest/spy": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.1.tgz", + "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==", + "dev": true, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/@xyflow/react": { "version": "12.11.3", "resolved": "https://registry.npmjs.org/@xyflow/react/-/react-12.11.3.tgz", @@ -2293,11 +2682,42 @@ "node": ">=8" } }, + "node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, + "node_modules/aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "dev": true, + "dependencies": { + "dequal": "^2.0.3" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/ast-types": { "version": "0.16.1", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.16.1.tgz", @@ -2336,6 +2756,15 @@ "node": ">=6.0.0" } }, + "node_modules/bidi-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.1.0.tgz", + "integrity": "sha512-fX1Onk0tdVPC7obPWB5EbJ1z7NVhLq4m2xZLq2YXBkxzMXIGRpNMU88n0EPgWseKl12J7zXs7qrDxPK4sRs2fg==", + "dev": true, + "dependencies": { + "require-from-string": "^2.0.2" + } + }, "node_modules/body-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", @@ -2501,6 +2930,15 @@ } ] }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "5.6.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", @@ -2766,6 +3204,25 @@ "node": ">= 8" } }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "dev": true, + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/css.escape": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", + "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", + "dev": true + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -2879,6 +3336,19 @@ "node": ">=12" } }, + "node_modules/data-urls": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", + "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "dev": true, + "dependencies": { + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/debounce-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-4.0.0.tgz", @@ -2909,6 +3379,12 @@ } } }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true + }, "node_modules/dedent": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", @@ -2975,6 +3451,15 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -2992,6 +3477,13 @@ "node": ">=0.3.1" } }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "peer": true + }, "node_modules/dot-prop": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", @@ -3078,6 +3570,18 @@ "node": ">=8.6" } }, + "node_modules/entities": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", + "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", + "dev": true, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -3110,6 +3614,12 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true + }, "node_modules/es-object-atoms": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", @@ -3146,6 +3656,15 @@ "node": ">=4" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -3198,6 +3717,15 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/express": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", @@ -3574,6 +4102,18 @@ "node": ">=16.9.0" } }, + "node_modules/html-encoding-sniffer": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", + "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", + "dev": true, + "dependencies": { + "@exodus/bytes": "^1.6.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -3639,6 +4179,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3764,6 +4313,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true + }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -3867,6 +4422,55 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsdom": { + "version": "29.0.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.0.0.tgz", + "integrity": "sha512-9FshNB6OepopZ08unmmGpsF7/qCjxGPbo3NbgfJAnPeHXnsODE9WWffXZtRFRFe0ntzaAOcSKNJFz8wiyvF1jQ==", + "dev": true, + "dependencies": { + "@asamuzakjp/css-color": "^5.0.1", + "@asamuzakjp/dom-selector": "^7.0.2", + "@bramus/specificity": "^2.4.2", + "@csstools/css-syntax-patches-for-csstree": "^1.1.1", + "@exodus/bytes": "^1.15.0", + "css-tree": "^3.2.1", + "data-urls": "^7.0.0", + "decimal.js": "^10.6.0", + "html-encoding-sniffer": "^6.0.0", + "is-potential-custom-element-name": "^1.0.1", + "lru-cache": "^11.2.7", + "parse5": "^8.0.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^6.0.1", + "undici": "^7.24.3", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^8.0.1", + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.1", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24.0.0" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsdom/node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "dev": true, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", @@ -4231,6 +4835,16 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "peer": true, + "bin": { + "lz-string": "bin/bin.js" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -4248,6 +4862,12 @@ "node": ">= 0.4" } }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "dev": true + }, "node_modules/media-typer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", @@ -4349,6 +4969,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/minimatch": { "version": "10.2.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", @@ -4462,6 +5091,19 @@ "node": ">= 10" } }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -4689,6 +5331,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "dev": true, + "dependencies": { + "entities": "^8.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -4733,9 +5387,9 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "engines": { "node": ">=12" }, @@ -4812,6 +5466,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "peer": true, + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, "node_modules/pretty-ms": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.0.tgz", @@ -4858,6 +5527,15 @@ "node": ">= 0.10" } }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/qs": { "version": "6.15.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", @@ -4937,6 +5615,13 @@ "react": "^19.2.8" } }, + "node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "peer": true + }, "node_modules/react-router": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", @@ -4988,6 +5673,19 @@ "node": ">= 4" } }, + "node_modules/redent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", + "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", + "dev": true, + "dependencies": { + "indent-string": "^4.0.0", + "strip-indent": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -5132,6 +5830,18 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", @@ -5330,6 +6040,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true + }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -5362,6 +6078,12 @@ "node": ">=0.10.0" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -5370,6 +6092,12 @@ "node": ">= 0.8" } }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true + }, "node_modules/stdin-discarder": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.2.2.tgz", @@ -5479,11 +6207,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strip-indent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", + "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, + "dependencies": { + "min-indent": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/style-mod": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==" }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true + }, "node_modules/systeminformation": { "version": "5.33.1", "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.33.1.tgz", @@ -5542,6 +6288,24 @@ "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==" }, + "node_modules/tinybench": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.1.4.tgz", + "integrity": "sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==", + "dev": true, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/tinyexec": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -5558,6 +6322,24 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tldts": { + "version": "7.4.13", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.13.tgz", + "integrity": "sha512-iHtaIWWIbMDkCeJdTBzZFGgbluE5J+oHlb2g7+oAz1S1gpuVpabRZdQyd471Vl8UUkcz2vXSL8xZH2kyCe8tfA==", + "dev": true, + "dependencies": { + "tldts-core": "^7.4.13" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.4.13", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.13.tgz", + "integrity": "sha512-mbYsrih5FRtGxs3Usvl/PqwJsNpp+jsmrdFviiK02teHDG0/HebBG/pqCylje3kzgXYzuLoHJF/0mz9W53t8Xg==", + "dev": true + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -5577,6 +6359,30 @@ "node": ">=0.6" } }, + "node_modules/tough-cookie": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz", + "integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==", + "dev": true, + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "dev": true, + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/ts-morph": { "version": "26.0.0", "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-26.0.0.tgz", @@ -5830,11 +6636,146 @@ } } }, + "node_modules/vitest": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.1.tgz", + "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", + "dev": true, + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/mocker": "5.0.1", + "chai": "^6.2.2", + "es-module-lexer": "^2.3.2", + "expect-type": "^1.4.0", + "magic-string": "^1.2.3", + "obug": "^2.1.4", + "picomatch": "^4.0.7", + "std-env": "^4.2.0", + "tinybench": "6.1.4", + "tinyexec": "1.3.0", + "tinyglobby": "^0.2.17", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^22.12.0 || ^24.0.0 || >=26.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "5.0.1", + "@vitest/browser-preview": "5.0.1", + "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", + "@vitest/coverage-istanbul": "5.0.1", + "@vitest/coverage-v8": "5.0.1", + "@vitest/ui": "5.0.1", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/vitest/node_modules/magic-string": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.1.tgz", + "integrity": "sha512-8lyCu36ErXR0J9uaGKlKQoiLZKmtI63YGLE8G2o9jyRPdr4X47LusSOwgOJOzcVtp81fTAAjxR7BwKz682Jhow==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, "node_modules/w3c-keyname": { "version": "2.2.8", "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==" }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "dev": true, + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-mimetype": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", + "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", + "dev": true, + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "dev": true, + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/which": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", @@ -5849,6 +6790,22 @@ "node": "^16.13.0 || >=18.0.0" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -5869,6 +6826,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 5094f86..79dcd1a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,6 +7,7 @@ "dev": "vite", "build": "tsc -b && vite build", "lint": "oxlint", + "test": "vitest run", "preview": "vite preview" }, "dependencies": { @@ -29,13 +30,18 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.3.3", + "@testing-library/jest-dom": "^7.0.1", + "@testing-library/react": "^16.3.3", + "@testing-library/user-event": "^14.6.7", "@types/node": "^24.13.3", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.4", + "jsdom": "^29.0.0", "oxlint": "^1.75.0", "tailwindcss": "^4.3.3", "typescript": "~6.0.2", - "vite": "^8.2.0" + "vite": "^8.2.0", + "vitest": "^5.0.1" } } diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 2cbf239..b80c1cd 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -12,11 +12,24 @@ import type { } from '@/types/auth' import type { Agent } from '@/types/agents' import type { Dataset } from '@/types/datasets' -import type { DesignImpact, DesignRevision, DesignSpec, Experiment, ExperimentResults, ExperimentRunResults, Replicate, Trial } from '@/types/experiments' +import type { + DesignImpact, + DesignRevision, + DesignSpec, + Experiment, + ExperimentResults, + ExperimentRunResults, + MeasurementCapabilities, + MeasurementPlan, + MetricRecommendationMetadata, + MeasurementPlanValidationReport, + Replicate, + Trial, +} from '@/types/experiments' import type { LLMConnectionCheck, LLMProvider, LLMSetting, LLMSettingModelsResponse } from '@/types/llmSettings' import type { McpServer } from '@/types/mcpServers' import type { OkfBundle, OkfDocument } from '@/types/okf' -import type { CellRunBatch, PromptPreview, Protocol, ProtocolGraph, ProtocolRevision, ProtocolRun } from '@/types/protocols' +import type { CellRunBatch, PromptPreview, Protocol, ProtocolGraph, ProtocolRevision, ProtocolRun, TestRun } from '@/types/protocols' import type { Run, RunStep } from '@/types/runs' import type { Skill, SkillListResponse, SkillUrlPreview } from '@/types/skills' @@ -172,18 +185,15 @@ export const experimentsApi = { list: (opts?: { includeArchived?: boolean }) => request(opts?.includeArchived ? '/experiments?include_archived=true' : '/experiments'), get: (id: string) => request(`/experiments/${id}`), - evaluationContext: (id: string, contextMetricIds: string[]) => - request<{ context: string }>(`/experiments/${id}/evaluation-context`, { - method: 'POST', - body: { context_metric_ids: contextMetricIds }, - }), - scoreCompletedRuns: (id: string) => - request<{ queued: number }>(`/experiments/${id}/score-completed-runs`, { method: 'POST' }), + validateMeasurementPlan: (id: string, data: { measurement_plan: MeasurementPlan | null; metrics: DesignSpec['metrics']; graph: ProtocolGraph }) => + request(`/experiments/${id}/measurement-plan/validate`, { method: 'POST', body: data }), + getMeasurementCapabilities: (id: string) => + request(`/experiments/${id}/measurement-capabilities`), // Omit `name` and the server allocates the next free "Untitled Experiment N" // atomically -- the one-click create in AppHeader relies on that, since a // name this client picks from a GET is a guess that another session (or an // archived experiment it can't see) can invalidate before the POST lands. - create: (data: { name?: string; description?: string | null }) => + create: (data: { name?: string; description?: string | null; measurement_plan?: MeasurementPlan | null }) => request('/experiments', { method: 'POST', body: data }), // Creates both a fresh experiment and its linked canvas in one server-side // transaction. Unlike the old canvas import, this never merges into the @@ -195,6 +205,7 @@ export const experimentsApi = { design_type?: string task_brief?: Record | null design_spec?: DesignSpec | null + measurement_plan?: MeasurementPlan | null graph: ProtocolGraph published_graph?: ProtocolGraph | null protocol_description?: string | null @@ -206,6 +217,9 @@ export const experimentsApi = { description?: string | null hypothesis?: string | null design_spec?: DesignSpec | null + measurement_plan?: MeasurementPlan | null + measurement_validation_protocol_id?: string | null + metric_recommendations?: MetricRecommendationMetadata | null // A timestamp to archive, null to unarchive -- canvas menu's Archive/Unarchive action. archived_at?: string | null // The experiment's whole attached-dataset list, in canvas wiring order @@ -248,7 +262,7 @@ export const experimentsApi = { // revision is superseded and a new one opened -- results for surviving cell // labels carry forward, the rest stay in history (see // services.design_generation). Nothing is ever deleted here. - generateDesign: (id: string, declaration?: { hypothesis?: string | null; design_spec?: DesignSpec | null }) => + generateDesign: (id: string, declaration?: { hypothesis?: string | null; design_spec?: DesignSpec | null; measurement_plan?: MeasurementPlan | null; measurement_validation_protocol_id?: string | null }) => request(`/experiments/${id}/generate-design`, { method: 'POST', body: declaration }), // One row per replicate (a "trial"), not per ProtocolRun -- a replicate that's never // been run is still listed, with status "not_started" (see TrialResponse / @@ -280,6 +294,8 @@ export const protocolsApi = { // instead of today's ad-hoc, un-substituted whole-graph run. run: (id: string, cellLabel?: string | null) => request(`/protocols/${id}/runs`, { method: 'POST', body: { replicate_label: cellLabel ?? null } }), + testRun: (id: string) => request(`/protocols/${id}/test-runs`, { method: 'POST' }), + getLatestTestRun: (id: string) => request(`/protocols/${id}/test-runs/latest`), // The per-node Play icon -- 422 if the node has upstream input or isn't a // runnable Agent (see validate_single_node_runnable). Same polling shape // as a plain run (getRun), just with node_runs carrying only this one key. @@ -294,7 +310,8 @@ export const protocolsApi = { getRun: (id: string, runId: string) => request(`/protocols/${id}/runs/${runId}`), // Only raises cancel_requested_at -- a no-op (200, unchanged row) once the // run is already terminal. run_protocol's own node loop is what actually - // honors it (between nodes, not mid-node) and flips status to "cancelled". + // honors it during task execution or metric evaluation and flips status to + // "cancelled" after retaining work that already completed. cancelRun: (id: string, runId: string) => request(`/protocols/${id}/runs/${runId}/cancel`, { method: 'POST' }), listRuns: (id: string) => request(`/protocols/${id}/runs`), // "Run all cells" -- 422 if there's no linked experiment or the graph diff --git a/frontend/src/components/CreateExperimentFromFileDialog.tsx b/frontend/src/components/CreateExperimentFromFileDialog.tsx index e2a227a..369d581 100644 --- a/frontend/src/components/CreateExperimentFromFileDialog.tsx +++ b/frontend/src/components/CreateExperimentFromFileDialog.tsx @@ -6,7 +6,7 @@ import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' -import type { DesignSpec } from '@/types/experiments' +import type { DesignSpec, MeasurementPlan } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' type JsonRecord = Record @@ -18,6 +18,7 @@ interface ImportedDefinition { designType: string taskBrief: Record | null designSpec: DesignSpec | null + measurementPlan: MeasurementPlan | null graph: ProtocolGraph publishedGraph: ProtocolGraph | null protocolDescription: string | null @@ -55,6 +56,11 @@ function parseDefinition(text: string): ImportedDefinition { designType: optionalString(experiment.design_type) ?? 'factorial', taskBrief: isRecord(experiment.task_brief) ? experiment.task_brief : null, designSpec: (isRecord(experiment.design_spec) ? experiment.design_spec : isRecord(parsed.design_spec) ? parsed.design_spec : null) as DesignSpec | null, + measurementPlan: (isRecord(experiment.measurement_plan) + ? experiment.measurement_plan + : isRecord(parsed.measurement_plan) + ? parsed.measurement_plan + : null) as unknown as MeasurementPlan | null, graph: parsed.graph as unknown as ProtocolGraph, publishedGraph, protocolDescription: optionalString(canvas.description) ?? optionalString(parsed.description), @@ -113,6 +119,7 @@ export function CreateExperimentFromFileDialog({ design_type: definition.designType, task_brief: definition.taskBrief, design_spec: definition.designSpec, + measurement_plan: definition.measurementPlan, graph: definition.graph, published_graph: definition.publishedGraph, protocol_description: definition.protocolDescription, diff --git a/frontend/src/components/protocol/AddNodePanel.tsx b/frontend/src/components/protocol/AddNodePanel.tsx index 1b7423c..fdb6de5 100644 --- a/frontend/src/components/protocol/AddNodePanel.tsx +++ b/frontend/src/components/protocol/AddNodePanel.tsx @@ -11,7 +11,7 @@ import { SKILL_BROWSE } from './skillCatalog' // tool picker, GET /datasets already backs the dataset picker, // services.protocol_execution._run_gated_worker already implements the // critic gate's revision loop, and _resolve_llm_config/_resolve_tool_config/ -// _resolve_dataset_configs/_resolve_script_config already resolve an agent's +// _resolve_dataset_configs/_resolve_script_configs already resolve an agent's // respective connectors. "memory" and the two pattern entries are the // exceptions -- each is real in the graph/validation sense (wiring one up is // accepted and does something visually) but has NO runtime effect yet, diff --git a/frontend/src/components/protocol/AgentNodeInspector.tsx b/frontend/src/components/protocol/AgentNodeInspector.tsx index 4c29846..e3745a8 100644 --- a/frontend/src/components/protocol/AgentNodeInspector.tsx +++ b/frontend/src/components/protocol/AgentNodeInspector.tsx @@ -19,7 +19,6 @@ import { PromptReferenceField } from './PromptReferenceField' import { useProtocolCanvasActions } from './ProtocolCanvasContext' import { RESIZE_HANDLE_CLASSNAME, useResizablePane } from './useResizablePane' import { experimentsApi } from '@/api/client' -import { normalizeDesignMetrics } from '@/lib/metricCatalog' import { referenceLabel, seedPromptText } from '@/lib/promptReferences' import type { HandoffPeers, PromptReferenceScope } from '@/lib/promptReferences' import type { AgentNodeConfig, AgentNodeData, NodeRunState, PromptPreview, ProtocolNode } from '@/types/protocols' @@ -133,18 +132,6 @@ export function AgentNodeInspector({ enabled: !!experimentId, }) - // Derived above the `!node` bail-out because the query below is a hook: it has - // to run on every render, including the no-selection one, where it's disabled. - const metrics = normalizeDesignMetrics(experimentQuery.data?.design_spec?.metrics) - const validMetricIds = new Set(metrics.map((metric) => metric.id!)) - const contextMetricIds = (node?.data.contextMetricIds ?? []).filter((id) => validMetricIds.has(id)) - const evaluationContextQuery = useQuery({ - queryKey: ['experiments', experimentId, 'evaluation-context', contextMetricIds], - queryFn: () => experimentsApi.evaluationContext(experimentId!, contextMetricIds), - enabled: !!experimentId && contextMetricIds.length > 0, - }) - const evaluationContext = evaluationContextQuery.data?.context ?? '' - if (!node) return null const data = node.data const config = data.config @@ -191,13 +178,6 @@ export function AgentNodeInspector({ onChange(node!.id, { ...data, factor_bindings: next }) } - function patchContextMetricIds(next: string[]) { - // Stale references disappear as soon as the node is next saved; the - // executor also filters them defensively, so a removed Design metric can - // never crash a run or leak old context. - onChange(node!.id, { ...data, contextMetricIds: next.filter((id) => validMetricIds.has(id)) }) - } - return ( -
-
- -

The Agent receives the selected metric definitions and optimization direction. Metric results are calculated or recorded after the run.

-
- {metrics.length === 0 ? ( -

No experiment metrics are available. Add metrics in Design.

- ) : ( - <> -
-
- {metrics.map((metric) => { - const checkboxId = `context-metric-${node.id}-${metric.id}` - return - })} -
- {evaluationContext &&
Preview generated context
{evaluationContext}
} - - )} -
diff --git a/frontend/src/components/protocol/CreateCustomMetricDialog.tsx b/frontend/src/components/protocol/CreateCustomMetricDialog.tsx new file mode 100644 index 0000000..52922b3 --- /dev/null +++ b/frontend/src/components/protocol/CreateCustomMetricDialog.tsx @@ -0,0 +1,116 @@ +import { useCallback, useRef, useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { experimentsApi } from '@/api/client' +import { agentOutputBindingForMetric } from '@/lib/agentOutputMetrics' +import { applyCustomMetricChange, type CustomMetricProducerConfig, type CustomMetricSourceContext } from '@/lib/customMetrics' +import { makeCustomMetric, normalizeDesignMetrics } from '@/lib/metricCatalog' +import { mcpToolBindingForMetric } from '@/lib/mcpToolMetrics' +import { pythonScriptBindingForMetric } from '@/lib/pythonScriptMetrics' +import type { ProtocolGraph } from '@/types/protocols' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { CustomMetricFlow } from './CustomMetricDialogs' +import { NODE_INSPECTOR_CONTENT_CLASSNAME } from './NodeInspectorDialog' + +export function CreateCustomMetricDialog({ + experimentId, + protocolId, + graph, + sourceContext, + metricId, + open, + onOpenChange, +}: { + experimentId: string + protocolId: string + graph: ProtocolGraph + sourceContext: CustomMetricSourceContext + metricId?: string + open: boolean + onOpenChange: (open: boolean) => void +}) { + const queryClient = useQueryClient() + const experimentQuery = useQuery({ + queryKey: ['experiments', experimentId], + queryFn: () => experimentsApi.get(experimentId), + enabled: open, + }) + const experiment = experimentQuery.data + const existingMetrics = normalizeDesignMetrics(experiment?.design_spec?.metrics) + const existingMetric = existingMetrics.find((metric) => metric.id === metricId) + const existingBinding = agentOutputBindingForMetric(experiment?.measurement_plan ?? null, metricId) + ?? pythonScriptBindingForMetric(experiment?.measurement_plan ?? null, metricId) + ?? mcpToolBindingForMetric(experiment?.measurement_plan ?? null, metricId) + const [draft] = useState(() => makeCustomMetric({ + name: '', + description: '', + })) + const flushAutosaveRef = useRef<(() => void) | null>(null) + const createMutation = useMutation({ + mutationFn: async ({ metric, config }: { metric: typeof draft; config: CustomMetricProducerConfig }) => { + // Re-read immediately before the replacement PATCH. Design metadata and + // its measurement plan are nested documents, so this preserves edits + // made elsewhere while the metric dialog was open. + const fresh = await experimentsApi.get(experimentId) + const currentMetrics = normalizeDesignMetrics(fresh.design_spec?.metrics) + const metrics = currentMetrics.some((candidate) => candidate.id === metric.id) + ? currentMetrics.map((candidate) => candidate.id === metric.id ? metric : candidate) + : [...currentMetrics, metric] + const measurementPlan = applyCustomMetricChange(fresh.measurement_plan, metric, config, graph) + return experimentsApi.update(experimentId, { + measurement_plan: measurementPlan, + measurement_validation_protocol_id: protocolId, + design_spec: { ...fresh.design_spec, metrics }, + }) + }, + onSuccess: (updated) => { + queryClient.setQueryData(['experiments', experimentId], updated) + queryClient.invalidateQueries({ queryKey: ['experiments'] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }, + }) + + const close = useCallback(() => { + flushAutosaveRef.current?.() + onOpenChange(false) + }, [onOpenChange]) + + return ( + next ? onOpenChange(true) : close()}> + + + {metricId ? 'Edit custom metric' : 'Create custom metric'} + {metricId ? 'Update this reported metric and its source.' : 'Capture an Agent output or tool result for display and export.'} + +
+ {experimentQuery.isPending ? ( +

Loading experiment metrics…

+ ) : experimentQuery.isError ? ( +

Could not load this experiment's metrics.

+ ) : metricId && !existingMetric ? ( +

This custom metric no longer exists.

+ ) : ( + { flushAutosaveRef.current = flush }} + onCancel={close} + onDirtyChange={() => undefined} + onSave={(metric, config) => createMutation.mutateAsync({ metric, config })} + /> + )} + {createMutation.isError && ( +

+ {createMutation.error instanceof Error ? createMutation.error.message : 'Could not create the custom metric.'} +

+ )} +
+ + + ) +} diff --git a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx new file mode 100644 index 0000000..ef6e26e --- /dev/null +++ b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx @@ -0,0 +1,136 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { describe, expect, it, vi } from 'vitest' +import type { DesignMetric } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { mcpServersApi } from '@/api/client' +import type { CustomMetricSourceContext } from '@/lib/customMetrics' +import { CustomMetricFlow } from './CustomMetricDialogs' + +function renderFlow(graph: ProtocolGraph, onSave = vi.fn(), sourceContext?: CustomMetricSourceContext) { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const metric: DesignMetric = { + id: 'metric-1', + name: '', + kind: 'custom', + valueType: 'number', + direction: 'neutral', + aggregation: 'mean', + primary: false, + } + render( + + + , + ) + return onSave +} + +describe('CustomMetricFlow', () => { + it('captures an Agent final output as a custom metric source', async () => { + const user = userEvent.setup() + const graph = { + nodes: [{ id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Judge' } }], + edges: [], + } as unknown as ProtocolGraph + const onSave = renderFlow(graph) + + await user.click(screen.getByRole('combobox', { name: 'Metric node' })) + await user.click(screen.getByRole('option', { name: 'Judge:Agent output' })) + await user.type(screen.getByRole('textbox', { name: 'Metric name' }), 'Quality') + await user.click(screen.getByRole('button', { name: 'Add custom metric' })) + + expect(onSave).toHaveBeenCalledWith( + expect.objectContaining({ name: 'Quality' }), + { producer: 'agent', agentNodeId: 'agent' }, + ) + }) + + it('captures a Python Script call without evaluator configuration', async () => { + const user = userEvent.setup() + const graph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer' } }, + { id: 'script', type: 'script', position: { x: 0, y: 0 }, data: { label: 'Scorer', config: { code: 'print(1)' } } }, + { id: 'dataset', type: 'dataset', position: { x: 0, y: 0 }, data: { label: 'Cases' } }, + ], + edges: [{ id: 'script-agent', source: 'script', target: 'agent', targetHandle: 'tool' }], + } as unknown as ProtocolGraph + const onSave = renderFlow(graph, vi.fn(), { producer: 'python', nodeId: 'script' }) + + expect(screen.queryByText('Evaluator')).not.toBeInTheDocument() + expect(screen.queryByText(/Python evaluators are trusted same-user code/)).not.toBeInTheDocument() + expect(screen.queryByRole('combobox', { name: 'Metric type' })).not.toBeInTheDocument() + expect(screen.queryByLabelText('Unit')).not.toBeInTheDocument() + expect(screen.queryByRole('textbox', { name: /Result path/ })).not.toBeInTheDocument() + await user.type(screen.getByRole('textbox', { name: 'Metric name' }), 'Quality') + await user.click(screen.getByRole('button', { name: 'Add custom metric' })) + + expect(onSave).toHaveBeenCalledWith( + expect.objectContaining({ name: 'Quality' }), + expect.objectContaining({ producer: 'python', sourceKey: JSON.stringify(['agent', 'script']) }), + ) + }) + + it('allows multiple metrics to capture the same MCP tool result', async () => { + const user = userEvent.setup() + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([{ + id: 'server-1', name: 'Quality server', transport: 'stdio', command: 'quality', url: null, status: 'connected', error_message: null, + capabilities: { tools: [{ name: 'score', input_schema: { properties: {} } }, { name: 'explain', input_schema: { properties: {} } }] }, created_at: '', + }]) + const graph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer' } }, + { id: 'tool', type: 'mcp_tool', position: { x: 0, y: 0 }, data: { label: 'Quality tools', config: { server_id: 'server-1', tool_names: ['score', 'explain'] } } }, + ], + edges: [{ id: 'tool-agent', source: 'tool', target: 'agent', targetHandle: 'tool' }], + } as unknown as ProtocolGraph + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + render() + + await user.click(screen.getByRole('combobox', { name: 'Metric node' })) + await user.click(screen.getByRole('option', { name: 'Writer:Quality tools:MCP Tool' })) + await waitFor(() => expect(screen.getByRole('combobox', { name: 'MCP tool' })).toHaveTextContent('score')) + await user.click(screen.getByRole('combobox', { name: 'MCP tool' })) + expect(screen.getByRole('option', { name: 'score' })).not.toHaveAttribute('aria-disabled', 'true') + }) + + it('allows multiple metrics to capture the same Python Script result', async () => { + const user = userEvent.setup() + const graph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer' } }, + { id: 'script', type: 'script', position: { x: 0, y: 0 }, data: { label: 'Scorer', config: { code: 'print(1)' } } }, + ], + edges: [{ id: 'script-agent', source: 'script', target: 'agent', targetHandle: 'tool' }], + } as unknown as ProtocolGraph + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + render() + + await user.click(screen.getByRole('combobox', { name: 'Metric node' })) + expect(screen.getByRole('option', { name: 'Writer:Scorer:Python Script' })).not.toHaveAttribute('aria-disabled', 'true') + }) +}) diff --git a/frontend/src/components/protocol/CustomMetricDialogs.tsx b/frontend/src/components/protocol/CustomMetricDialogs.tsx new file mode 100644 index 0000000..28bba62 --- /dev/null +++ b/frontend/src/components/protocol/CustomMetricDialogs.tsx @@ -0,0 +1,268 @@ +import { useEffect, useRef, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { mcpServersApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { AGENT_OUTPUT_PRODUCER_ID, agentOutputSourceOptions } from '@/lib/agentOutputMetrics' +import { mcpToolSourceOptions, type McpToolSourceOption } from '@/lib/mcpToolMetrics' +import { pythonScriptSourceOptions } from '@/lib/pythonScriptMetrics' +import type { CustomMetricProducer, CustomMetricProducerConfig, CustomMetricSourceContext } from '@/lib/customMetrics' +import type { DesignMetric, MeasurementPlan } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { useDialogAutosave, type DialogAutosaveStatus } from './useDialogAutosave' + +type Binding = MeasurementPlan['producers'][number] | undefined + +function duplicateName(metric: DesignMetric, name: string, metrics: DesignMetric[]) { + const normalized = name.trim().toLocaleLowerCase() + return metrics.some((item) => item.id !== metric.id && item.name.trim().toLocaleLowerCase() === normalized) +} + +function availableMetricName(metric: DesignMetric, name: string, metrics: DesignMetric[]) { + const requested = name.trim() + const base = metrics.find((item) => item.id !== metric.id && item.name.trim().toLocaleLowerCase() === requested.toLocaleLowerCase())?.name.trim() || requested || 'Metric' + const names = new Set(metrics.filter((item) => item.id !== metric.id).map((item) => item.name.trim().toLocaleLowerCase())) + let suffix = 2 + while (names.has(`${base} ${suffix}`.toLocaleLowerCase())) suffix += 1 + return `${base} ${suffix}` +} + +function autosaveLabel(status: DialogAutosaveStatus, invalid: boolean) { + if (invalid) return 'Complete the required fields to start autosaving.' + if (status === 'waiting') return 'Waiting to save…' + if (status === 'saving') return 'Saving…' + if (status === 'error') return 'Could not autosave. Edit a field or close to retry.' + if (status === 'saved') return 'Saved' + return 'Changes save automatically.' +} + +export function CustomMetricFlow({ metric, binding, graph, existingMetrics, sourceContext, submitting = false, autosave = false, onAutosaveReady, onCancel, onDirtyChange, onSave }: { + metric: DesignMetric + binding: Binding + graph: ProtocolGraph | undefined + existingMetrics: DesignMetric[] + sourceContext?: CustomMetricSourceContext + submitting?: boolean + autosave?: boolean + onAutosaveReady?: (flush: (() => void) | null) => void + onCancel?: () => void + onDirtyChange: (dirty: boolean) => void + onSave: (metric: DesignMetric, config: CustomMetricProducerConfig) => void | Promise +}) { + const agentSources = agentOutputSourceOptions(graph).filter((source) => !sourceContext || sourceContext.producer === 'agent' && source.agentNodeId === sourceContext.nodeId) + const pythonSources = pythonScriptSourceOptions(graph).filter((source) => !sourceContext || sourceContext.producer === 'python' && source.scriptNodeId === sourceContext.nodeId) + const mcpSources = mcpToolSourceOptions(graph).filter((source) => !sourceContext || sourceContext.producer === 'mcp' && source.mcpNodeId === sourceContext.nodeId) + const initialProducer: CustomMetricProducer | undefined = binding?.producer_id === AGENT_OUTPUT_PRODUCER_ID + ? 'agent' + : binding?.producer_id === 'asaree.python_script' + ? 'python' + : binding?.producer_id === 'asaree.mcp_tool' + ? 'mcp' + : sourceContext?.producer + const initialAgentSource = agentSources.find((source) => source.agentNodeId === binding?.config.agent_node_id) + ?? (sourceContext?.producer === 'agent' && agentSources.length === 1 ? agentSources[0] : undefined) + const initialPythonSource = pythonSources.find((source) => source.agentNodeId === binding?.config.agent_node_id && source.scriptNodeId === binding?.config.script_node_id) + ?? (sourceContext?.producer === 'python' && pythonSources.length === 1 ? pythonSources[0] : undefined) + const initialMcpSource = mcpSources.find((source) => source.agentNodeId === binding?.config.agent_node_id && source.mcpNodeId === binding?.config.mcp_node_id) + ?? (sourceContext?.producer === 'mcp' && mcpSources.length === 1 ? mcpSources[0] : undefined) + const editing = existingMetrics.some((item) => item.id === metric.id) + const [name, setName] = useState(editing ? metric.name : '') + const [producer, setProducer] = useState(initialProducer) + const [agentNodeId, setAgentNodeId] = useState(initialAgentSource?.agentNodeId ?? '') + const serversQuery = useQuery({ queryKey: ['mcp-servers'], queryFn: mcpServersApi.list, staleTime: 60_000, enabled: mcpSources.length > 0 }) + const servers = serversQuery.data ?? [] + const [pythonSourceKey, setPythonSourceKey] = useState(initialPythonSource?.key ?? '') + const [mcpSourceKey, setMcpSourceKey] = useState(initialMcpSource?.key ?? '') + const [toolName, setToolName] = useState(typeof binding?.config.tool_name === 'string' ? binding.config.tool_name : '') + const selectedPythonSource = pythonSources.find((source) => source.key === pythonSourceKey) + const selectedMcpSource = mcpSources.find((source) => source.key === mcpSourceKey) + const selectedServer = servers.find((server) => server.id === selectedMcpSource?.serverId) + const selectedTool = selectedServer?.capabilities?.tools?.find((tool) => tool.name === toolName) + const duplicate = duplicateName(metric, name, existingMetrics) + const detailError = !name.trim() ? 'Enter a metric name.' : duplicate ? `A metric with this name already exists. Try ${availableMetricName(metric, name, existingMetrics)}.` : undefined + const agentSourceDisabledReason = (source: (typeof agentSources)[number]) => source.disabledReason + const selectedAgentSource = agentSources.find((source) => source.agentNodeId === agentNodeId) + const agentSourceError = !selectedAgentSource ? 'Choose an Agent.' : agentSourceDisabledReason(selectedAgentSource) + const pythonSourceDisabledReason = (source: (typeof pythonSources)[number]) => source.disabledReason + const pythonSourceError = !selectedPythonSource + ? 'Choose a direct Agent-to-Python-Script connection.' + : pythonSourceDisabledReason(selectedPythonSource) + const mcpSourceDisabledReason = (source: McpToolSourceOption) => source.disabledReason + ?? (serversQuery.isError ? 'Registered MCP Servers could not be loaded.' + : serversQuery.isPending ? undefined + : !servers.some((server) => server.id === source.serverId) ? 'The registered MCP Server is unavailable.' + : servers.find((server) => server.id === source.serverId)?.status !== 'connected' ? 'The registered MCP Server is not connected.' + : undefined) + const mcpSourceError = !selectedMcpSource + ? 'Choose a direct Agent-to-MCP Tool connection.' + : mcpSourceDisabledReason(selectedMcpSource) + const selectedMetricNode = selectedAgentSource + ? `agent:${selectedAgentSource.agentNodeId}` + : selectedPythonSource + ? `python:${selectedPythonSource.key}` + : selectedMcpSource + ? `mcp:${selectedMcpSource.key}` + : '__none__' + const sourceNodeLabel = (nodeId: string) => graph?.nodes.find((node) => node.id === nodeId)?.data.label || nodeId + const sourceAgentLabel = (agentNodeId: string) => graph?.nodes.find((node) => node.id === agentNodeId)?.data.label || agentNodeId + // Keep this picker in the same agent:node:field form as the factor picker. + // The connection itself is the selectable field here, so its source type + // is the final segment. + const pythonNodeLabel = (source: (typeof pythonSources)[number]) => `${sourceAgentLabel(source.agentNodeId)}:${sourceNodeLabel(source.scriptNodeId)}:Python Script` + const mcpNodeLabel = (source: McpToolSourceOption) => `${sourceAgentLabel(source.agentNodeId)}:${sourceNodeLabel(source.mcpNodeId)}:MCP Tool` + const agentNodeLabel = (source: (typeof agentSources)[number]) => `${source.label}:Agent output` + const selectedNodeLabel = selectedAgentSource + ? agentNodeLabel(selectedAgentSource) + : selectedPythonSource + ? pythonNodeLabel(selectedPythonSource) + : selectedMcpSource + ? mcpNodeLabel(selectedMcpSource) + : 'Select a node…' + const sourceSelected = producer === 'agent' ? Boolean(selectedAgentSource) + : producer === 'python' ? Boolean(selectedPythonSource) + : producer === 'mcp' ? Boolean(selectedMcpSource) : false + function selectMetricNode(value: string | null) { + const agentSource = agentSources.find((source) => `agent:${source.agentNodeId}` === value) + if (agentSource) { + setProducer('agent') + setAgentNodeId(agentSource.agentNodeId) + setPythonSourceKey('') + setMcpSourceKey('') + setToolName('') + return + } + const pythonSource = pythonSources.find((source) => `python:${source.key}` === value) + if (pythonSource) { + setProducer('python') + setAgentNodeId('') + setPythonSourceKey(pythonSource.key) + setMcpSourceKey('') + setToolName('') + return + } + const mcpSource = mcpSources.find((source) => `mcp:${source.key}` === value) + if (mcpSource) { + setProducer('mcp') + setAgentNodeId('') + setPythonSourceKey('') + setMcpSourceKey(mcpSource.key) + setToolName('') + return + } + setProducer(undefined) + setAgentNodeId('') + setPythonSourceKey('') + setMcpSourceKey('') + setToolName('') + } + const selectedToolDisabled = Boolean(toolName && selectedMcpSource && !selectedMcpSource.toolNames.includes(toolName)) + const mcpMappingError = selectedToolDisabled + ? 'The selected MCP tool is disabled for this MCP node.' + : !selectedTool + ? 'Choose an available MCP tool.' + : undefined + const initialSignature = JSON.stringify({ + name: editing ? metric.name : '', + producer: initialProducer, + agentNodeId: initialAgentSource?.agentNodeId ?? '', + pythonSourceKey: initialPythonSource?.key ?? '', + mcpSourceKey: initialMcpSource?.key ?? '', + toolName: typeof binding?.config.tool_name === 'string' ? binding.config.tool_name : '', + }) + const currentSignature = JSON.stringify({ name, producer, agentNodeId, pythonSourceKey, mcpSourceKey, toolName }) + const formRef = useRef(null) + useEffect(() => onDirtyChange(currentSignature !== initialSignature), [currentSignature, initialSignature, onDirtyChange]) + useEffect(() => { + formRef.current?.scrollIntoView?.({ behavior: 'smooth', block: 'start' }) + }, []) + + useEffect(() => { + if (producer !== 'mcp' || toolName) return + const availableTool = selectedMcpSource?.toolNames[0] + if (availableTool) setToolName(availableTool) + }, [producer, selectedMcpSource, toolName]) + + const saveDisabled = Boolean( + detailError + || !producer + || (producer === 'agent' ? agentSourceError : producer === 'python' ? pythonSourceError : mcpSourceError || mcpMappingError), + ) + const nextMetric = { ...metric, name: name.trim(), description: metric.description, direction: 'neutral' as const, valueType: 'opaque' as const, aggregation: 'none' as const, primary: false } + const nextConfig: CustomMetricProducerConfig | null = producer === 'agent' && selectedAgentSource + ? { producer, agentNodeId: selectedAgentSource.agentNodeId } + : producer === 'mcp' && selectedMcpSource + ? { producer, source: selectedMcpSource, toolName } + : producer === 'python' + ? { producer, sourceKey: pythonSourceKey } + : null + const autosaveDraft = autosave && !saveDisabled && nextConfig && currentSignature !== initialSignature + ? { metric: nextMetric, config: nextConfig } + : null + const metricAutosave = useDialogAutosave({ + draft: autosaveDraft, + signature: currentSignature, + onSave: ({ metric: savedMetric, config }) => onSave(savedMetric, config), + }) + useEffect(() => { + if (!autosave || !onAutosaveReady) return + onAutosaveReady(metricAutosave.flush) + return () => onAutosaveReady(null) + }, [autosave, metricAutosave.flush, onAutosaveReady]) + + return
+
+

{editing ? `Edit ${metric.name}` : 'Create custom metric'}

+

Choose an Agent final output or Agent tool result to capture and export.

+
+ +
+ Metric node + {!sourceContext ?
+ + +
: <> +

{producer === 'agent' ? 'Agent output' : producer === 'python' ? 'Python Script' : 'MCP Tool'}

+ {producer === 'agent' &&

{selectedAgentSource?.label}

{agentSourceError &&

{agentSourceError}

}
} + {producer === 'python' &&
{pythonSources.length === 1 ?

{selectedPythonSource?.label}

: }{pythonSourceError && }
} + {producer === 'mcp' &&
{mcpSources.length === 1 ?

{selectedMcpSource?.label}

: }{mcpSourceError && }
} + } + {producer === 'mcp' &&

{serversQuery.isPending ? 'Loading registered MCP Servers…' : serversQuery.isError ? 'Registered MCP Servers could not be loaded.' : 'Registered MCP Server availability loaded.'}

} +
+ + {sourceSelected &&
+ Metric details +
setName(event.target.value)} />{detailError && }
+
} + + {sourceSelected && producer === 'mcp' &&
+
{selectedTool?.description &&

{selectedTool.description}

}
+ {mcpMappingError &&

{mcpMappingError}

} +

ASAREE records this tool's last call result. The Agent decides whether and how often to call it.

+
} + + {sourceSelected && producer === 'agent' &&

ASAREE records this Agent's completed final output verbatim.

} + + {autosave ? ( +

+ {autosaveLabel(submitting ? 'saving' : metricAutosave.status, saveDisabled)} +

+ ) : ( +
+ + +
+ )} +
+} diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index 71c7473..ff6b22a 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -1,16 +1,15 @@ import { useCallback, useEffect, useRef, useState, type RefObject } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' -import { Info, Pencil, Plus, X } from 'lucide-react' +import { Pencil, Plus, X } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' -import { Checkbox } from '@/components/ui/checkbox' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { Textarea } from '@/components/ui/textarea' -import { experimentsApi, llmSettingsApi, protocolsApi } from '@/api/client' +import { experimentsApi, protocolsApi } from '@/api/client' import { coordinationStrategyIssues } from '@/lib/coordinationStrategy' import { unboundFactorNames } from '@/lib/factorBindings' import { promptReferenceScope } from '@/lib/promptReferences' @@ -25,19 +24,9 @@ import { import { LEVEL_TYPE_LABELS, levelTypeOf } from './factorLevels' import { FactorEditorDialog } from './FactorEditorDialog' import { InfoTooltip } from './InfoTooltip' -import { ModelField } from './ModelField' -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' -import { useProviderModels } from './useProviderModels' -import { - METRIC_CATALOG, - makeCatalogMetric, - makeCustomMetric, - metricCatalogEntry, - normalizeDesignMetrics, - type MetricAggregation, - type MetricDirection, - type MetricValueType, -} from '@/lib/metricCatalog' +import { MetricsEditor } from './MetricsEditor' +export { MetricsEditor } from './MetricsEditor' +import { normalizeDesignMetrics } from '@/lib/metricCatalog' import type { ProtocolCanvasHandle } from './ProtocolCanvas' import type { ProtocolEdge, ProtocolGraph, ProtocolNode } from '@/types/protocols' import { @@ -46,9 +35,8 @@ import { type DesignFactor, type DesignMetric, type Experiment, - type MetricScoringConfig, + type MeasurementPlan, } from '@/types/experiments' -import { LLM_PROVIDER_CATALOG, LLM_PROVIDER_LABELS, type LLMProvider } from '@/types/llmSettings' const AUTOSAVE_DELAY_MS = 800 @@ -71,6 +59,7 @@ type MetadataDraft = { hypothesis: string randomizationSeed: number | null metrics: DesignMetric[] + measurementPlan: MeasurementPlan | null coordinationSlug: CoordinationStrategySlug } @@ -170,7 +159,7 @@ function AddFactorButton({ // its same pre-existing risk of racing a concurrent unsaved edit // elsewhere on this tab (see FactorEditorDialog.tsx's design notes). const fresh = await experimentsApi.get(experiment.id) - const nextFactors = [...(fresh.design_spec?.factors ?? []), factor] + const nextFactors = [...(fresh.design_spec?.factors ?? []).filter((candidate) => candidate.name !== factor.name), factor] await experimentsApi.update(experiment.id, { design_spec: { ...fresh.design_spec, factors: nextFactors } }) return { factor, field } }, @@ -178,7 +167,6 @@ function AddFactorButton({ canvasRef.current?.bindFactor(field.nodeId, field.fieldPath, factor.name) queryClient.invalidateQueries({ queryKey: ['experiments', experiment.id] }) queryClient.invalidateQueries({ queryKey: ['experiments', experiment.id, 'design-impact'] }) - setDialogOpen(false) }, }) @@ -200,7 +188,7 @@ function AddFactorButton({ pickableFields={fields} existingNames={existingNames} onSave={(factor, field) => { - if (field) createMutation.mutate({ factor, field }) + if (field) return createMutation.mutateAsync({ factor, field }) }} /> )} @@ -275,7 +263,6 @@ function FactorsEditor({ if (next.name !== oldName) canvasRef.current?.renameFactorBindings(oldName, next.name) queryClient.invalidateQueries({ queryKey: ['experiments', experiment.id] }) queryClient.invalidateQueries({ queryKey: ['experiments', experiment.id, 'design-impact'] }) - setEditingFactor(null) }, }) @@ -331,220 +318,13 @@ function FactorsEditor({ // has to be recovered from the canvas's own factor_bindings. boundField={factorBoundField(graphQuery.data?.nodes ?? [], editingFactor.draft.name)} promptScopeFor={promptScopeFor} - onSave={(next) => editMutation.mutate({ oldName: editingFactor.draft.name, next })} + onSave={(next) => editMutation.mutateAsync({ oldName: editingFactor.draft.name, next })} /> )} ) } -function withPrimary(metrics: DesignMetric[]): DesignMetric[] { - if (!metrics.length) return [] - const selected = metrics.findIndex((metric) => metric.primary) - return metrics.map((metric, index) => ({ ...metric, primary: index === (selected < 0 ? 0 : selected) })) -} - -function MetricInfo({ metric }: { metric: DesignMetric }) { - const catalog = metricCatalogEntry(metric.catalogKey) - return ( - - - }> - - - - {metric.description || catalog?.shortDescription || 'Metric definition unavailable.'} - {catalog ? `${catalog.kind.replace(/_/g, ' ')} metric` : `${metric.kind ?? 'custom'} metric`} - - - - ) -} - -function CustomMetricDialog({ - metric, - existingNames, - onCancel, - onSave, -}: { - metric?: DesignMetric - existingNames: string[] - onCancel: () => void - onSave: (metric: { name: string; description: string; direction: MetricDirection; valueType: MetricValueType; aggregation: MetricAggregation; unit?: string; scoring?: MetricScoringConfig }) => void -}) { - const [name, setName] = useState(metric?.name ?? '') - const [description, setDescription] = useState(metric?.description ?? '') - const [direction, setDirection] = useState(metric?.direction ?? 'maximize') - const [valueType, setValueType] = useState(metric?.valueType ?? 'number') - const [aggregation, setAggregation] = useState(metric?.aggregation ?? 'mean') - const [unit, setUnit] = useState(metric?.unit ?? '') - const [judgeEnabled, setJudgeEnabled] = useState(metric?.scoring?.method === 'model_judge') - const [rubric, setRubric] = useState(metric?.scoring?.rubric ?? '') - const [reference, setReference] = useState(metric?.scoring?.reference ?? '') - const [minimum, setMinimum] = useState(metric?.scoring?.min?.toString() ?? '0') - const [maximum, setMaximum] = useState(metric?.scoring?.max?.toString() ?? '100') - const [judgeProvider, setJudgeProvider] = useState(metric?.scoring?.judge?.provider ?? '') - const [judgeModel, setJudgeModel] = useState(metric?.scoring?.judge?.model ?? '') - const credentialsQuery = useQuery({ queryKey: ['llm-settings'], queryFn: () => llmSettingsApi.list(), staleTime: 10 * 60 * 1000 }) - const { modelsQuery, models } = useProviderModels(judgeProvider || undefined) - const registeredProviders = (credentialsQuery.data ?? []).map((credential) => credential.provider) - const normalizedName = name.trim().toLocaleLowerCase() - const duplicate = !!normalizedName && existingNames.some((existing) => existing.trim().toLocaleLowerCase() === normalizedName && existing !== metric?.name) - const parsedMinimum = minimum.trim() === '' ? undefined : Number(minimum) - const parsedMaximum = maximum.trim() === '' ? undefined : Number(maximum) - const invalidRange = judgeEnabled && ( - !rubric.trim() - || (parsedMinimum !== undefined && !Number.isFinite(parsedMinimum)) - || (parsedMaximum !== undefined && !Number.isFinite(parsedMaximum)) - || (parsedMinimum !== undefined && parsedMaximum !== undefined && parsedMinimum > parsedMaximum) - ) - const unavailableJudgeCredential = !!judgeProvider && !credentialsQuery.isLoading && !registeredProviders.includes(judgeProvider) - const invalidJudge = judgeEnabled && (!judgeProvider || !judgeModel.trim() || unavailableJudgeCredential) - const invalid = !name.trim() || !description.trim() || duplicate || invalidRange || invalidJudge - return ( - !open && onCancel()}> - - - {metric ? 'Edit custom metric' : 'Custom metric'} - Custom metrics are reporting targets by default. Enable an LLM judge below to score each completed output automatically. - -
-
setName(event.target.value)} />
-