diff --git a/CHANGELOG.md b/CHANGELOG.md index dca386b..302a52a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,46 @@ - OpenCode edits are no longer gated either: the OpenCode plugin runs the same `pre_edit` hook, which now always allows the edit. The plugin does not yet show the reminder; that rework comes with the edit-bridge removal. - `install-claude-hooks` and `uninstall-claude-hooks` refuse to touch a `~/.claude/settings.json` that is not valid JSON, instead of replacing it. - The manual `hooks/daem0n_*.py` scripts are deprecated stubs that print a notice and exit 0, and `hooks/settings.json.example` is removed. If you installed them by hand, re-run `python -m daem0nmcp.cli install-claude-hooks`; it also removes the old entries. +- **Behaviour change — installed profiles now turn themselves on.** An unset (or + empty) `DAEM0NMCP__ENABLED` means "auto": the profile is ready when + its extra is installed and disabled otherwise. `false` still turns it off, and + `true` with packages missing is still degraded. An empty value used to mean + `false`. What this costs if you installed an extra and never set the flag: + `[graph]` imports networkx, igraph and leidenalg at startup, adds a graph + provider to recall (so rankings change) and lets dreaming refresh communities; + `[local]`/`[models-local]` import sentence-transformers and onnxruntime at + startup, download the embedding model on first use, and rebuild the dense + projection, which re-embeds the whole workspace once. Opt out per profile with + `DAEM0NMCP__ENABLED=false`. +- `sandbox_execute_python` now honours `DAEM0NMCP_AGENCY_E2B_ENABLED=false`; it + previously required only `E2B_API_KEY`. +- `system_health` and `meta.capability_states` report the profile's real + remediation (install command or environment variable) instead of "Review the + <name> capability profile." +- `start_daem0nmcp_server.bat` serves the directory you run it from (or its first + argument, or an exported `DAEM0NMCP_PROJECT_ROOT`) instead of the Daem0n-MCP + checkout, and `install-opencode` installs the four `.opencode/commands/` files. +- **Migration payload change — re-migrate a store migrated by an earlier + v7 development build.** `migrate-v7` no longer copies a v6 host path into the + event it writes: the columns that hold one (`file_path`, `project_path`, + `database_path`, and a `file_path_relative` that is not workspace-relative) + are recorded as a `{"$host_path": {...}}` digest, so an export bundle from a + migrated workspace no longer discloses the origin host's directory layout. + The raw value stays in the retained v6 tables and in + `.daem0nmcp/storage/migrations/v7//source.snapshot.db`. Because the + digest changes the lossless payload, a store migrated before this build fails + `verify-v7` with "retained legacy source provenance differs" and must be + migrated again from its intact v6 source (`migrate-v7 --rollback`, then + `migrate-v7 --apply`). v7 is unreleased, so this affects development stores + only. +- **Migrated v6 workspaces are usable end to end.** `session_brief`, recall, + outcomes, the four resources and `workspace_export` all work on a migrated + store; `migrate-v7` reports in `warnings` whatever it could not copy + faithfully; an un-migrated v6 workspace answers the new non-retryable + `MIGRATION_REQUIRED` naming the command instead of a retryable error with no + remedy; and `migrate-v7 --rollback` refuses with `ROLLBACK_WOULD_HIDE_WRITES` + rather than silently stranding writes made since activation (pass + `--discard-v7-writes` to accept that). ## [6.0.0] - 2026-01-29 diff --git a/README.md b/README.md index c2ca8d7..216a6d5 100644 --- a/README.md +++ b/README.md @@ -1037,9 +1037,23 @@ python -m daem0nmcp.cli install-opencode --force This regenerates `opencode.json` and the plugin to match the latest version. -### 5. Migrations Run Automatically +### 5. Migrations -Database schema migrations are applied automatically when any MCP tool runs. No manual migration step required. +Within a storage format, schema migrations are applied automatically when any +MCP tool runs, and no manual step is required. + +Moving an existing v6 workspace to the v7 storage format is **not** automatic. +It is an offline, reversible command, and a v6 workspace answers +`MIGRATION_REQUIRED` until it has run: + +```bash +# Stop the server first, then: +python -m daem0nmcp.cli --project-path /path/to/project migrate-v7 --apply + +# Undo it; refuses if events were recorded since activation unless +# --discard-v7-writes is passed. +python -m daem0nmcp.cli --project-path /path/to/project migrate-v7 --rollback +``` ### 6. Index Your Codebase diff --git a/daem0nmcp/api/v7/application.py b/daem0nmcp/api/v7/application.py index 2104068..75a76f3 100644 --- a/daem0nmcp/api/v7/application.py +++ b/daem0nmcp/api/v7/application.py @@ -15,7 +15,7 @@ from ...covenant import CovenantGate, InvocationScope from ...workspace import Workspace -from .errors import ErrorCode +from .errors import DATABASE_IN_USE_RETRY_AFTER_MS, ErrorCode from .models import CapabilityState from .policy import V7_TOOL_LEVELS from .responses import ResponseFactory @@ -119,18 +119,26 @@ def _remedy( if code in {"COUNSEL_REQUIRED", "TOKEN_MISSING"} and isinstance( workspace_id, str ): - target_arguments = { - key: value - for key, value in effective.items() - if key not in {"workspace_id", "preflight_token"} - } - return "memory_preflight", { - "workspace_id": workspace_id, - "target_tool": tool_name, - "target_arguments": target_arguments, - } + return self._preflight_remedy(workspace_id, tool_name, effective) return None, {} + @staticmethod + def _preflight_remedy( + workspace_id: str, + tool_name: str, + effective: Mapping[str, Any], + ) -> tuple[str, dict[str, Any]]: + target_arguments = { + key: value + for key, value in effective.items() + if key not in {"workspace_id", "preflight_token"} + } + return "memory_preflight", { + "workspace_id": workspace_id, + "target_tool": tool_name, + "target_arguments": target_arguments, + } + def handler(self, tool_name: str) -> Callable[..., Any]: try: input_model = TOOL_INPUT_MODELS[tool_name] @@ -294,6 +302,24 @@ async def invoke(**arguments: Any) -> object: states = () if any(not isinstance(state, CapabilityState) for state in states): states = () + if error_code == ErrorCode.DATABASE_IN_USE.value: + # The gate already spent any preflight token, so a + # retry needs a fresh one. + workspace_id = effective.get("workspace_id") + remedy_tool, remedy_arguments = ( + self._preflight_remedy(workspace_id, tool_name, effective) + if isinstance(token, str) and isinstance(workspace_id, str) + else (None, {}) + ) + return response.failure( + ErrorCode.DATABASE_IN_USE, + "The workspace database is currently in use.", + retryable=True, + retry_after_ms=DATABASE_IN_USE_RETRY_AFTER_MS, + remedy_tool=remedy_tool, + remedy_arguments=remedy_arguments, + capability_states=tuple(states), + ) return response.failure( ErrorCode(error_code), "The operation could not be completed.", diff --git a/daem0nmcp/api/v7/code_entity_operations.py b/daem0nmcp/api/v7/code_entity_operations.py index 5368379..e9a3daf 100644 --- a/daem0nmcp/api/v7/code_entity_operations.py +++ b/daem0nmcp/api/v7/code_entity_operations.py @@ -49,7 +49,7 @@ from .models import ( EvidenceRef, RecordSummary, - contains_absolute_filesystem_path, + stored_relative_path, ) from .public_ids import ( PublicObjectIdNotFound, @@ -79,7 +79,6 @@ _MAX_ENTITY_EVENTS = 200 _MAX_IMPACT_ENTITIES = 500 _MAX_IMPACT_EDGES = 5_000 -_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _EVENT_TYPE_RE = re.compile(r"^[a-z][a-z0-9_.-]{2,79}$") _EVENT_COLUMNS = ( "event_id,workspace_id,stream_id,stream_kind,stream_version,event_type," @@ -366,7 +365,6 @@ def _validated_record_row( or row["stream_version"] != expected_version or row["state_hash"] != memory_state_hash(state) or row["content_hash"] != memory_content_hash(state) - or row["file_path"] is not None ): raise CodeEntityOperationError("CAPABILITY_DEGRADED") return row @@ -382,17 +380,17 @@ def _event_record_summary( content = state["content"] if not isinstance(content, str) or not content: raise CodeEntityOperationError("CAPABILITY_DEGRADED") + # A record's valid time may precede or follow the transaction time it was + # written at; ``RecordSummary`` shows the earlier of the two. created = _datetime_from_us(occurred_at_us) updated = _datetime_from_us(recorded_at_us) - if created > updated: - raise CodeEntityOperationError("CAPABILITY_DEGRADED") try: return RecordSummary( record_id=record_id, record_type=state["record_type"], excerpt=content[:4000], tags=state["tags"], - relative_file_path=state["file_path_relative"], + relative_file_path=stored_relative_path(state["file_path_relative"]), current_status=( "invalidated" if state["deleted_at_us"] is not None @@ -935,12 +933,8 @@ def _entity_row( def _safe_event_content(state: Mapping[str, Any]) -> str: content = state.get("content") - if ( - not isinstance(content, str) - or not content - or _CONTROL_RE.search(content) is not None - or contains_absolute_filesystem_path(content) - ): + # Stored user text reads back as stored (UD-3). + if not isinstance(content, str) or not content: raise CodeEntityOperationError("CAPABILITY_DEGRADED") return content diff --git a/daem0nmcp/api/v7/discovery_operations.py b/daem0nmcp/api/v7/discovery_operations.py index 0b9a22d..d345d1f 100644 --- a/daem0nmcp/api/v7/discovery_operations.py +++ b/daem0nmcp/api/v7/discovery_operations.py @@ -16,7 +16,6 @@ import inspect import json import os -import posixpath import re import secrets import sqlite3 @@ -51,7 +50,13 @@ ) from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import CapabilityState, Page, RecordSummary, RetrievalData +from .models import ( + CapabilityState, + Page, + RecordSummary, + RetrievalData, + stored_relative_path, +) from .public_ids import ( PublicObjectIdNotFound, PublicObjectIdRepository, @@ -755,24 +760,6 @@ def _partition_metadata( return row_count, content_hash -def _safe_code_path(value: object) -> bool: - if ( - not isinstance(value, str) - or not 1 <= len(value) <= 1024 - or "\\" in value - or "\x00" in value - or value.startswith(("/", "~")) - or re.match(r"^[A-Za-z]:", value) is not None - or value in {".", ".."} - ): - return False - components = value.split("/") - return ( - all(component not in {"", ".", ".."} for component in components) - and posixpath.normpath(value) == value - ) - - def _validate_entity_partition( connection: sqlite3.Connection, workspace_id: str, @@ -1061,24 +1048,24 @@ def _record_summary(row: sqlite3.Row) -> RecordSummary: try: content = row["content"] tags = json.loads(str(row["tags_json"])) + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote and + # may carry a valid time ahead of its transaction time; neither is + # corruption, and ``RecordSummary`` bounds what is emitted. if ( not isinstance(content, str) or not content or not isinstance(tags, list) - or row["file_path"] is not None or row["deleted_at_us"] is not None ): raise ValueError created = _datetime_from_us(row["created_at_us"]) updated = _datetime_from_us(row["updated_at_us"]) - if updated < created: - raise ValueError return RecordSummary( record_id=row["record_id"], record_type=row["record_type"], excerpt=content[:4000], tags=tags, - relative_file_path=row["file_path_relative"], + relative_file_path=stored_relative_path(row["file_path_relative"]), current_status="archived" if bool(row["archived"]) else "current", content_hash=row["content_hash"], created_at=created, @@ -1424,7 +1411,10 @@ def _code_capability_error( else "CAPABILITY_DEGRADED" ) remediation = ( - "Set DAEM0NMCP_APPS_ENABLED=true and install the apps dependency profile." + "Set DAEM0NMCP_APPS_ENABLED to true, false, or leave it unset." + if capability_status == "failed" + else "Install the apps profile with pip install 'daem0nmcp[apps]'; " + "it turns on automatically unless DAEM0NMCP_APPS_ENABLED=false." ) return DiscoveryOperationError( code, @@ -2046,6 +2036,56 @@ def reader(connection: sqlite3.Connection) -> _EntitySelection: raise _translate_error(error) from None +def _lexical_catchup_pending_sync( + dependencies: DiscoveryOperationDependencies, + workspace: Workspace, + record_ids: tuple[str, ...], +) -> bool: + """Return whether the lexical projection still owes these records. + + True only when the records are absent from the active lexical generation + *and* a rebuild job is queued or running, i.e. a retry can actually make + progress. A dead-lettered rebuild, a permanently unavailable index, or a + record that simply does not match the entity name is not retryable. + """ + + if not record_ids: + return False + + def reader(connection: sqlite3.Connection) -> bool: + active = connection.execute( + "SELECT generation FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND status='active'", + (workspace.workspace_id,), + ).fetchone() + indexed = 0 + if active is not None: + placeholders = ",".join("?" for _ in record_ids) + indexed = int( + connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + f"AND projection_generation=? AND record_id IN ({placeholders})", + (workspace.workspace_id, int(active[0]), *record_ids), + ).fetchone()[0] + ) + if indexed >= len(record_ids): + return False + return ( + connection.execute( + "SELECT 1 FROM background_jobs WHERE workspace_id=? " + "AND job_type='retrieval.projection_rebuild' " + "AND status IN ('queued','running') LIMIT 1", + (workspace.workspace_id,), + ).fetchone() + is not None + ) + + try: + return bool(_read_snapshot(dependencies, workspace, reader)) + except Exception: + return False + + def _generation_is_current_sync( dependencies: DiscoveryOperationDependencies, workspace: Workspace, @@ -2093,7 +2133,21 @@ async def _memory_recall_entity( raise DiscoveryOperationError("CAPABILITY_DEGRADED") indexed[record_id] = item.record if set(indexed) != set(selection.record_ids): - raise DiscoveryOperationError("CAPABILITY_DEGRADED") + missing = tuple( + record_id + for record_id in selection.record_ids + if record_id not in indexed + ) + # Only a projection that has genuinely not caught up yet is worth + # retrying. A permanently unavailable or dead-lettered lexical + # index stays a terminal CAPABILITY_DEGRADED. + pending = await _run_blocking( + dependencies, + lambda: _lexical_catchup_pending_sync(dependencies, workspace, missing), + ) + raise DiscoveryOperationError( + "DATABASE_IN_USE" if pending else "CAPABILITY_DEGRADED" + ) await _run_blocking( dependencies, lambda: _generation_is_current_sync( diff --git a/daem0nmcp/api/v7/errors.py b/daem0nmcp/api/v7/errors.py index c42fe14..9f1a7c1 100644 --- a/daem0nmcp/api/v7/errors.py +++ b/daem0nmcp/api/v7/errors.py @@ -6,9 +6,12 @@ from __future__ import annotations +import sqlite3 from enum import Enum from types import MappingProxyType +from ...bounded_workers import BoundedWorkerBusyError + class ErrorCode(str, Enum): """Reviewed business-error codes returned inside ``ApiResponse``.""" @@ -22,6 +25,7 @@ class ErrorCode(str, Enum): STALE_PROJECTION_ID = "STALE_PROJECTION_ID" CAPABILITY_DISABLED = "CAPABILITY_DISABLED" CAPABILITY_DEGRADED = "CAPABILITY_DEGRADED" + MIGRATION_REQUIRED = "MIGRATION_REQUIRED" LEXICAL_UNAVAILABLE = "LEXICAL_UNAVAILABLE" COMMUNION_REQUIRED = "COMMUNION_REQUIRED" COUNSEL_REQUIRED = "COUNSEL_REQUIRED" @@ -56,6 +60,15 @@ class ErrorCode(str, Enum): # caller; server-side logs may contain the corresponding private details. INTERNAL_ERROR_MESSAGE = "Internal error." +# A retained v6 store cannot be read by v7 and retrying never helps. The +# remedy is an offline command, not an MCP tool, so it is stated in the message +# with a placeholder instead of a machine-readable ``remedy``. +MIGRATION_REQUIRED_MESSAGE = ( + "This workspace still uses the v6 storage format. Stop the server, then " + "run: python -m daem0nmcp.cli --project-path migrate-v7 " + "--apply" +) + def is_stable_error_code(value: object) -> bool: """Return whether *value* is an exact reviewed v7 error code.""" @@ -63,11 +76,45 @@ def is_stable_error_code(value: object) -> bool: return isinstance(value, str) and value in STABLE_ERROR_CODE_SET +# Clients back off this long before retrying DATABASE_IN_USE. +DATABASE_IN_USE_RETRY_AFTER_MS = 250 + + +def is_database_busy(error: BaseException | None) -> bool: + """Return whether *error* is transient contention a caller may retry. + + A full worker pool and a SQLite lock or busy timeout both clear once the + competing writer finishes, so they map to retryable ``DATABASE_IN_USE``. + The ``code`` check keeps an already-classified repository error (which is + not a ``RuntimeServiceError``) busy through the briefing/preflight wrappers. + """ + + if isinstance(error, BoundedWorkerBusyError): + return True + if getattr(error, "code", None) == ErrorCode.DATABASE_IN_USE.value: + return True + if not isinstance(error, sqlite3.OperationalError): + return False + code = getattr(error, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in {5, 6} # SQLITE_BUSY, SQLITE_LOCKED + # sqlite_errorcode is unavailable on Python 3.10; match SQLite's fixed + # lock diagnostics only, never arbitrary detail text. + return str(error).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } + + __all__ = [ + "DATABASE_IN_USE_RETRY_AFTER_MS", "ERROR_CODE_REGISTRY", "INTERNAL_ERROR_MESSAGE", + "MIGRATION_REQUIRED_MESSAGE", "STABLE_ERROR_CODES", "STABLE_ERROR_CODE_SET", "ErrorCode", + "is_database_busy", "is_stable_error_code", ] diff --git a/daem0nmcp/api/v7/external_operations.py b/daem0nmcp/api/v7/external_operations.py index 57d8001..f07abcb 100644 --- a/daem0nmcp/api/v7/external_operations.py +++ b/daem0nmcp/api/v7/external_operations.py @@ -12,6 +12,7 @@ from types import MappingProxyType from typing import Any, Protocol +from ...capabilities import CapabilityRegistry from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET @@ -528,6 +529,29 @@ async def sandbox_execute_python( ), ), ) + capability = CapabilityRegistry(environ=dependencies.environment).get( + "agency-e2b" + ) + if capability["status"] != "ready": + # The advertised opt-out must also switch off remote code execution. + raise ExternalOperationError( + "CAPABILITY_DISABLED" + if capability["status"] == "disabled" + else "CAPABILITY_DEGRADED", + capability_states=( + CapabilityState( + name="agency-e2b", + status=capability["status"], + reason_code="CAPABILITY_DISABLED" + if capability["status"] == "disabled" + else "CAPABILITY_DEGRADED", + remediation=( + "Install 'daem0nmcp[agency-e2b]' and leave " + "DAEM0NMCP_AGENCY_E2B_ENABLED unset or true." + ), + ), + ), + ) provider = ( dependencies.sandbox_provider_factory() if dependencies.sandbox_provider_factory diff --git a/daem0nmcp/api/v7/intelligence_operations.py b/daem0nmcp/api/v7/intelligence_operations.py index c144850..cdd4644 100644 --- a/daem0nmcp/api/v7/intelligence_operations.py +++ b/daem0nmcp/api/v7/intelligence_operations.py @@ -34,8 +34,8 @@ from .models import ( EvidenceRef, RecordSummary, - contains_absolute_filesystem_path, parse_wire_datetime, + stored_relative_path, ) from .public_ids import PublicObjectIdNotFound, PublicObjectIdRepository from .resources import RuleView @@ -63,7 +63,6 @@ _MAX_COMMUNITY_MEMBERS = 20_000 _CLAIM_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|[\r\n]+") _WORD_RE = re.compile(r"[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*") -_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _NEGATIONS = frozenset({"no", "not", "never", "none", "without"}) _STOP_WORDS = frozenset( { @@ -369,12 +368,8 @@ def _verified_event(row: sqlite3.Row) -> dict[str, Any]: def _safe_text(value: object) -> str: - if ( - not isinstance(value, str) - or not value - or _CONTROL_RE.search(value) is not None - or contains_absolute_filesystem_path(value) - ): + # Stored user text reads back as stored (UD-3). + if not isinstance(value, str) or not value: raise IntelligenceOperationError("CAPABILITY_DEGRADED") return value @@ -442,7 +437,7 @@ def _record_evidence_from_events( content = _safe_text(state.get("content")) try: tags = state.get("tags", []) - relative_path = state.get("file_path_relative") + relative_path = stored_relative_path(state.get("file_path_relative")) content_hash = memory_content_hash(state) recorded_at_us = int(row["recorded_at_us"]) created = _datetime_from_us(created_at_us) diff --git a/daem0nmcp/api/v7/local_state_operations.py b/daem0nmcp/api/v7/local_state_operations.py index c222eb7..2dc7cc4 100644 --- a/daem0nmcp/api/v7/local_state_operations.py +++ b/daem0nmcp/api/v7/local_state_operations.py @@ -42,6 +42,7 @@ MutationReceipt, RecordSummary, parse_wire_datetime, + stored_relative_path, ) from .resources import ActiveContextItem from .runtime_services import WorkspaceStorageResolver @@ -277,8 +278,8 @@ def _json_string_list(value: object) -> list[str]: def _record_summary(row: sqlite3.Row) -> RecordSummary: - if row["file_path"] is not None: - raise LocalStateOperationError("CAPABILITY_DEGRADED") + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; it + # is legacy provenance, never emitted, so it must not deny the read. content = row["content"] record_type = row["record_type"] archived = row["archived"] @@ -305,7 +306,7 @@ def _record_summary(row: sqlite3.Row) -> RecordSummary: "record_type": record_type, "excerpt": content[:4000], "tags": _json_string_list(row["tags_json"]), - "relative_file_path": row["file_path_relative"], + "relative_file_path": stored_relative_path(row["file_path_relative"]), "current_status": status, "content_hash": row["content_hash"], "created_at": _datetime_from_us(row["created_at_us"]), diff --git a/daem0nmcp/api/v7/maintenance_operations.py b/daem0nmcp/api/v7/maintenance_operations.py index 2a07ee6..fcaa689 100644 --- a/daem0nmcp/api/v7/maintenance_operations.py +++ b/daem0nmcp/api/v7/maintenance_operations.py @@ -35,7 +35,12 @@ from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import DestructiveMutationReceipt, Preview, RecordSummary +from .models import ( + DestructiveMutationReceipt, + Preview, + RecordSummary, + stored_relative_path, +) from .runtime_protocols import ActiveStorageResolver, WorkerPool from .runtime_services import WorkspaceStorageResolver from .tasks import await_task_terminal @@ -300,9 +305,10 @@ def _verified_snapshot(row: sqlite3.Row) -> _RecordSnapshot: try: state = _record_state(row) payload = _parse_json(row["payload_json"], dict) + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; + # ``_record_state`` already drops it from the compared state. if ( - row["file_path"] is not None - or row["workspace_id"] != row["event_workspace_id"] + row["workspace_id"] != row["event_workspace_id"] or row["record_id"] != row["event_stream_id"] or payload.get("record") != state or canonical_json_bytes(payload).decode("utf-8") != str(row["payload_json"]) @@ -672,7 +678,9 @@ def _record_summary(record: _RecordSnapshot) -> RecordSummary: "record_type": record.state["record_type"], "excerpt": content[:4000], "tags": tags, - "relative_file_path": record.state["file_path_relative"], + "relative_file_path": stored_relative_path( + record.state["file_path_relative"] + ), "current_status": status, "content_hash": record.content_hash, "created_at": created_at, diff --git a/daem0nmcp/api/v7/models.py b/daem0nmcp/api/v7/models.py index c440e7e..e1a44aa 100644 --- a/daem0nmcp/api/v7/models.py +++ b/daem0nmcp/api/v7/models.py @@ -8,9 +8,10 @@ from __future__ import annotations import json -import posixpath import re +from collections.abc import Iterator, Mapping from datetime import datetime, timezone +from functools import cache from typing import ( TYPE_CHECKING, Annotated, @@ -19,6 +20,7 @@ Literal, TypeAlias, TypeVar, + get_args, ) from pydantic import ( @@ -33,6 +35,7 @@ ) from typing_extensions import Self, TypeAliasType +from ...workspace import is_workspace_relative_path from .errors import INTERNAL_ERROR_MESSAGE, ErrorCode MAX_JSON_COLLECTION_ITEMS = 4096 @@ -46,7 +49,6 @@ r"[0-9]{2}:[0-9]{2}:[0-9]{2}" r"(?:\.[0-9]{1,9})?(?:Z|[+-][0-9]{2}:[0-9]{2})$" ) -_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:") _WINDOWS_ABSOLUTE_PATH = re.compile( r"(? bool: - """Return whether a nested public value contains an absolute path.""" +def is_host_absolute_path(value: str) -> bool: + """Return whether text contains an absolute host path or file URI. + + This is the only absolute-path predicate on the v7 wire. ``WireModel`` + applies it to every string except ``UserText`` fields. + """ + + return bool( + _WINDOWS_ABSOLUTE_PATH.search(value) + or _POSIX_ABSOLUTE_PATH.search(value) + or _FILE_URI.search(value) + ) + + +class _Exempt: + """``Annotated`` marker for strings the substring path scan skips.""" + + __slots__ = ("name",) + + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return self.name + + +_USER_TEXT = _Exempt("UserText") +# RelativePath's own validator refuses absolute and escaping paths exactly; +# the prose scan would also refuse valid names such as ``src/__pycache__/x``. +_RELATIVE_PATH = _Exempt("RelativePath") + +# Memory content, queries, rule text and server text quoting them may mention +# routes and file paths ("/health", "C:\proj\app.py"). Fields typed with it, +# alone or inside a list, set or dict, are exempt from the absolute-path rule in +# both directions, so anything accepted on input can always be read back. +UserText = Annotated[str, _USER_TEXT] + + +def _carries(annotation: object, marker: _Exempt) -> bool: + return annotation is marker or any( + _carries(argument, marker) for argument in get_args(annotation) + ) + + +def _fields_marked(model: type[BaseModel], marker: _Exempt) -> frozenset[str]: + return frozenset( + name + for name, field in model.model_fields.items() + if marker in field.metadata or _carries(field.annotation, marker) + ) + + +@cache +def _user_text_fields(model: type[BaseModel]) -> frozenset[str]: + return _fields_marked(model, _USER_TEXT) + + +@cache +def _exempt_fields(model: type[BaseModel]) -> frozenset[str]: + return _user_text_fields(model) | _fields_marked(model, _RELATIVE_PATH) + + +def guarded_strings(value: object, user_text: bool = False) -> Iterator[str]: + """Yield every string, keys included, that the absolute-path rule covers. + + Nested models are walked with their own field annotations, so a model + inside a ``UserText`` container is still checked. + """ - if isinstance(value, str): - return bool( - _WINDOWS_ABSOLUTE_PATH.search(value) - or _POSIX_ABSOLUTE_PATH.search(value) - or _FILE_URI.search(value) - ) - if isinstance(value, dict): - return any( - contains_absolute_filesystem_path(key) - or contains_absolute_filesystem_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(contains_absolute_filesystem_path(item) for item in value) if isinstance(value, BaseModel): - return contains_absolute_filesystem_path(value.__dict__) - return False + exempt = _exempt_fields(type(value)) + for name, item in value.__dict__.items(): + yield from guarded_strings(item, name in exempt) + elif isinstance(value, Mapping): + for key, item in value.items(): + yield from guarded_strings(key, user_text) + yield from guarded_strings(item, user_text) + elif isinstance(value, (list, tuple, set, frozenset)): + for item in value: + yield from guarded_strings(item, user_text) + elif isinstance(value, str) and not user_text: + yield value + + +def contains_absolute_filesystem_path(value: object) -> bool: + """Return whether a nested value has a guarded string with a host path.""" + + return any(is_host_absolute_path(text) for text in guarded_strings(value)) def _relative_path(value: str) -> str: - if value == ".": - return value - if ( - not value - or "\\" in value - or "\x00" in value - or value.startswith(("/", "~")) - or _WINDOWS_DRIVE.match(value) is not None - ): - raise ValueError("path must be a normalized workspace-relative POSIX path") - components = value.split("/") - if any(component in {"", ".", ".."} for component in components): - raise ValueError("path must be a normalized workspace-relative POSIX path") - if posixpath.normpath(value) != value: + if not is_workspace_relative_path(value): raise ValueError("path must be a normalized workspace-relative POSIX path") return value @@ -283,6 +340,7 @@ def _document_validation_error( str, StringConstraints(strict=True, min_length=1, max_length=1024), AfterValidator(_relative_path), + _RELATIVE_PATH, ] AwareDateTime = Annotated[datetime, BeforeValidator(parse_wire_datetime)] UtcDateTime = Annotated[AwareDateTime, AfterValidator(_utc)] @@ -326,7 +384,8 @@ def _document_validation_error( Field(max_length=MAX_JSON_COLLECTION_ITEMS), ], ) -ContextJsonObject = Annotated[JsonObject, AfterValidator(_context_size)] +UserJsonObject = Annotated[JsonObject, _USER_TEXT] +ContextJsonObject = Annotated[UserJsonObject, AfterValidator(_context_size)] ErrorCodeValue = Annotated[ErrorCode, BeforeValidator(_error_code)] UpperSnakeCode = Annotated[ @@ -348,10 +407,14 @@ def _document_validation_error( ), ] Tag = Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=80), - AfterValidator(_sanitized), ] +# Every stored tag list. ``RecordSummary.tags`` requires uniqueness on the +# way out, so the inputs that feed it require it on the way in; without it a +# plain ``memory_store(tags=['a','a'])`` wrote a record no reader could +# render. +UniqueTags = Annotated[list[Tag], AfterValidator(_unique_strings)] ProviderName = Annotated[ str, StringConstraints( @@ -430,7 +493,7 @@ class WireModel(BaseModel): @model_validator(mode="after") def reject_absolute_filesystem_paths(self) -> WireModel: - if contains_absolute_filesystem_path(self.__dict__): + if contains_absolute_filesystem_path(self): raise ValueError("absolute filesystem paths are forbidden on the v7 wire") return self @@ -471,7 +534,8 @@ class FieldError(WireModel): class ErrorRemedy(WireModel): tool: ToolName - arguments: JsonObject = Field(default_factory=dict) + # The caller's own validated arguments, echoed back for the retry. + arguments: UserJsonObject = Field(default_factory=dict) Remedy = ErrorRemedy @@ -582,13 +646,51 @@ class Page(WireModel, Generic[T]): truncated: bool +MIGRATED_EMPTY_CONTENT = "" + + +def stored_relative_path(value: object) -> str | None: + """Return a stored path only when it is a workspace-relative POSIX path. + + v6 stored an absolute ``file_path`` plus a relative form that is + ``../outside/x.py`` for a file above the project (and the absolute path + itself across Windows drives). Neither may leave the workspace, so a + migrated record reads back without its file link instead of failing. + """ + + if not isinstance(value, str): + return None + try: + return _relative_path(value) + except ValueError: + return None + + class RecordSummary(WireModel): + """The bounded public view of a stored record. + + It is only ever built from storage, never parsed from a caller, so it + clips rows that predate v7's bounds -- v6 tags had no length, uniqueness + or count limit, and v6 content could be empty -- rather than denying every + read of a migrated workspace. A v7-written record satisfies those three, + because every stored tag input is ``UniqueTags`` and content has its own + minimum. + + The fourth clip is not a v6-only repair: a record's creation carries its + valid time, which a backdated or future ``happened_at`` legitimately puts + on either side of the transaction time it was written at, so the summary + shows the earlier of the two rather than refusing the record. + + ``relative_file_path`` is a containment rule rather than a bound, so + it stays refused here; a stored value is normalized by + ``stored_relative_path`` where the row is read. + """ + record_id: RecordId record_type: RecordType excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=4000), - AfterValidator(_sanitized), ] tags: Annotated[list[Tag], AfterValidator(_unique_strings)] = Field( default_factory=list, @@ -600,11 +702,37 @@ class RecordSummary(WireModel): created_at: AwareDateTime updated_at: AwareDateTime - @model_validator(mode="after") - def validate_timeline(self) -> RecordSummary: - if self.updated_at < self.created_at: - raise ValueError("updated_at cannot precede created_at") - return self + @model_validator(mode="before") + @classmethod + def clip_stored_record(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + values = dict(data) + tags = values.get("tags") + if isinstance(tags, (list, tuple)): + clipped: list[str] = [] + for tag in tags: + if not isinstance(tag, str) or not tag: + continue + tag = tag[:80] + if tag not in clipped: + clipped.append(tag) + values["tags"] = clipped[:32] + excerpt = values.get("excerpt") + if isinstance(excerpt, str): + values["excerpt"] = excerpt[:4000] or MIGRATED_EMPTY_CONTENT + # A record's creation carries its valid time, which may be backdated + # or in the future relative to the transaction time it was written at, + # and v6 kept the two independently. The summary shows the earlier of + # the two as the creation rather than refusing the record. + try: + created = parse_wire_datetime(values["created_at"]) + updated = parse_wire_datetime(values["updated_at"]) + except (KeyError, TypeError, ValueError): + return values + if updated < created: + values["created_at"] = updated + return values class EvidenceRef(WireModel): @@ -621,9 +749,8 @@ class EvidenceItem(WireModel): citation: Citation record: RecordSummary bounded_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8000), - AfterValidator(_sanitized), ] channels: Annotated[list[ProviderName], AfterValidator(_unique_strings)] = Field( min_length=1, @@ -694,7 +821,7 @@ class RetrievalData(WireModel): items: list[EvidenceItem] = Field(default_factory=list, max_length=50) rendered_context: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] | None @@ -790,6 +917,10 @@ class Preview(WireModel): "EventId", "FactId", "FieldError", + "guarded_strings", + "is_host_absolute_path", + "MIGRATED_EMPTY_CONTENT", + "stored_relative_path", "JsonObject", "JsonValue", "MAX_CONTEXT_JSON_BYTES", @@ -819,9 +950,12 @@ class Preview(WireModel): "RuleId", "SelectionToken", "Tag", + "UniqueTags", "TokenUsage", "ToolName", "TriggerId", + "UserJsonObject", + "UserText", "UtcDateTime", "VersionId", "WireModel", diff --git a/daem0nmcp/api/v7/operations.py b/daem0nmcp/api/v7/operations.py index abb1cff..456b377 100644 --- a/daem0nmcp/api/v7/operations.py +++ b/daem0nmcp/api/v7/operations.py @@ -11,6 +11,7 @@ import asyncio import hashlib import hmac +import logging import os import re import secrets @@ -24,6 +25,8 @@ from types import MappingProxyType from typing import Any +from pydantic import TypeAdapter, ValidationError + from ...bounded_workers import BoundedWorkerBusyError, BoundedWorkerPool from ...covenant import CovenantGate, InvocationScope from ...event_store import ( @@ -50,8 +53,14 @@ ) from ...workspace import Workspace from .application import AdmittedRequest -from .errors import STABLE_ERROR_CODE_SET -from .models import EvidenceRef, Page, RecordSummary +from .errors import STABLE_ERROR_CODE_SET, is_database_busy +from .models import ( + EvidenceRef, + Page, + RecordSummary, + RelativePath, + stored_relative_path, +) from .portable_projections import ( ImportFinalizationLease, PortableTransferError, @@ -89,8 +98,11 @@ _MAX_EXPORT_EVENTS = 10_000 _IMPORT_LEASE_RENEW_MARGIN_US = 5 * 60 * 1_000_000 _CURSOR_RE = re.compile(r"^cur_([0-9a-f]{64})$") -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? str: + """Name a transfer failure the caller can act on, and log its cause. + + A concurrent writer (a projection drain or a dreaming job) can hold the + SQLite write lock past the busy timeout while a transfer does its session + bookkeeping. That is a transient, retryable condition, not an invalid + bundle, and reporting it as one leaves the caller with no next step. + Only the explicit ``raise ... from`` chain counts: a bundle rejected while + an unrelated busy error is in flight is still an invalid bundle. + """ + + seen: set[int] = set() + cause: BaseException | None = error + while cause is not None and id(cause) not in seen: + seen.add(id(cause)) + if is_database_busy(cause): + _LOGGER.warning("%s deferred: %s", operation, cause) + return "DATABASE_IN_USE" + cause = cause.__cause__ + _LOGGER.warning( + "%s rejected the bundle: %s", operation, type(error).__name__, exc_info=error + ) + return "IMPORT_INVALID" + + def _verify_schema(connection: sqlite3.Connection) -> None: try: row = connection.execute( @@ -240,6 +289,9 @@ def _active_connection( try: with DatabaseFileLock(storage, "shared"): active = resolve_active_database(storage) + if active.format_version == 6: + # An intact but un-migrated v6 store: retrying never helps. + raise CoreOperationError("MIGRATION_REQUIRED") if active.format_version != _FORMAT_VERSION: raise CoreOperationError("CAPABILITY_DEGRADED") connection = sqlite3.connect(active.path, timeout=5.0) @@ -447,22 +499,42 @@ def _validate_bundle(bundle: Mapping[str, Any], workspace_id: str) -> None: def _reject_raw_paths(value: object) -> None: - if isinstance(value, str): - if ( - _WINDOWS_ABSOLUTE_PATH.search(value) is not None - or _POSIX_ABSOLUTE_PATH.search(value) is not None - ): - raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - elif isinstance(value, Mapping): + """Refuse host paths in the structured path fields of an event payload. + + Free text (content, rationale, rule text, context) is user text and may + mention paths, as on the wire. + """ + if isinstance(value, Mapping): for key, item in value.items(): - if ( - key in {"file_path", "project_path", "database_path"} - and item is not None - and item != "" - ): + if key in _RAW_PATH_KEYS and item is not None and item != "": raise CoreOperationError("WORKSPACE_PATH_ESCAPE") - _reject_raw_paths(item) + if key in _RELATIVE_PATH_KEYS and item is not None: + try: + _RELATIVE_PATH_ADAPTER.validate_python(item) + except ValidationError: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") from None + if key != "context": + _reject_raw_paths(item) elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + # A migrated event carries its v6 row as ``[name, value]`` pairs, so a + # path column is a list element rather than a mapping key. Check that + # shape too: the migration replaces these with a digest, and a + # regression must fail here rather than reach the wire. + if ( + len(value) == 2 + # A column name is a string; anything else here is an ordinary + # two-element list, whose members may be unhashable. + and isinstance(value[0], str) + and value[0] in _PATH_PAIR_NAMES + and isinstance(value[1], str) + and value[1] + ): + if value[0] in _RAW_PATH_KEYS: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") + try: + _RELATIVE_PATH_ADAPTER.validate_python(value[1]) + except ValidationError: + raise CoreOperationError("WORKSPACE_PATH_ESCAPE") from None for item in value: _reject_raw_paths(item) elif isinstance(value, (bytes, bytearray, memoryview)): @@ -628,7 +700,7 @@ def _summary( record = _record_from_event(event) content = record.get("content") tags = record.get("tags", []) - relative = record.get("file_path_relative") + relative = stored_relative_path(record.get("file_path_relative")) if not isinstance(content, str) or not content or not isinstance(tags, list): raise CoreOperationError("IMPORT_INVALID") if record.get("deleted_at_us") is not None: @@ -912,6 +984,8 @@ def _projection_sync( raise CoreOperationError("CAPABILITY_DISABLED") from exc if exc.code in {"LEXICAL_UNAVAILABLE", "FTS5_UNAVAILABLE"}: raise CoreOperationError("LEXICAL_UNAVAILABLE") from exc + if exc.code == "DATABASE_IN_USE": + raise CoreOperationError("DATABASE_IN_USE") from exc raise CoreOperationError("CAPABILITY_DEGRADED") from exc except CoreOperationError: raise @@ -993,6 +1067,8 @@ def _export_sync( content = page["content"] manifest = page["manifest"] items = content["items"] + if page["page_kind"] == "legacy": + _reject_raw_paths(items) descriptor = page.get( "page_descriptor", { @@ -1028,9 +1104,14 @@ def _export_sync( except CoreOperationError: raise except PortableTransferError as exc: - raise CoreOperationError(exc.code) from exc + code = exc.code + if code == "IMPORT_INVALID": + code = _portable_failure_code(exc, "workspace_export") + raise CoreOperationError(code) from exc except Exception as exc: - raise CoreOperationError("IMPORT_INVALID") from exc + raise CoreOperationError( + _portable_failure_code(exc, "workspace_export") + ) from exc def _journal_payload(bundle: ExportBundle, merge: bool) -> dict[str, Any]: @@ -1051,6 +1132,7 @@ def _portable_page(bundle: ExportBundle) -> dict[str, Any]: items = [event.model_dump(mode="json") for event in bundle.events] elif bundle.page_kind == "legacy": items = list(bundle.legacy_rows) + _reject_raw_paths(items) else: items = [point.model_dump(mode="json") for point in bundle.vector_points] return { @@ -1128,6 +1210,35 @@ def _event_command(event: Mapping[str, Any]) -> EventCommand: ) +def _abandon_finalization_lease(storage: Path, lease: ImportFinalizationLease) -> None: + with _ABANDONED_FINALIZATION_LEASES_LOCK: + _ABANDONED_FINALIZATION_LEASES[(str(storage), lease.session_id)] = lease + + +def _release_abandoned_finalization( + connection: sqlite3.Connection, + storage: Path, + session_id: str, + *, + now: datetime, +) -> None: + """Finish an ended attempt's lease release before claiming the session. + + A still-held lock raises here and reaches the caller as DATABASE_IN_USE, + so the lease stays recorded for the next retry. + """ + + key = (str(storage), session_id) + with _ABANDONED_FINALIZATION_LEASES_LOCK: + abandoned = _ABANDONED_FINALIZATION_LEASES.get(key) + if abandoned is None: + return + release_import_finalization(connection, storage, abandoned, now=now) + with _ABANDONED_FINALIZATION_LEASES_LOCK: + if _ABANDONED_FINALIZATION_LEASES.get(key) is abandoned: + del _ABANDONED_FINALIZATION_LEASES[key] + + def _import_v2_sync( dependencies: CoreOperationDependencies, workspace: Workspace, @@ -1222,6 +1333,12 @@ def _import_v2_sync( "skipped": receipt.imported + receipt.skipped, } ) + _release_abandoned_finalization( + connection, + storage, + session_id, + now=dependencies.clock(), + ) lease = claim_import_finalization( connection, storage, @@ -1446,7 +1563,10 @@ def finalization_checkpoint() -> None: if vector_candidate is not None: vector_candidate.discard(connection, workspace.workspace_id) vector_candidate = None - raise CoreOperationError(exc.code) from exc + code = exc.code + if code == "IMPORT_INVALID": + code = _portable_failure_code(exc, "workspace_import") + raise CoreOperationError(code) from exc except CoreOperationError: if connection.in_transaction: connection.rollback() @@ -1474,16 +1594,20 @@ def finalization_checkpoint() -> None: if vector_candidate is not None: vector_candidate.discard(connection, workspace.workspace_id) vector_candidate = None - raise CoreOperationError("IMPORT_INVALID") from exc + raise CoreOperationError( + _portable_failure_code(exc, "workspace_import") + ) from exc finally: if lease is not None and not committed: - with suppress(Exception): + try: release_import_finalization( connection, storage, lease, now=dependencies.clock(), ) + except Exception: + _abandon_finalization_lease(storage, lease) def _import_sync( diff --git a/daem0nmcp/api/v7/pinned.py b/daem0nmcp/api/v7/pinned.py index 9b612ba..43a01e4 100644 --- a/daem0nmcp/api/v7/pinned.py +++ b/daem0nmcp/api/v7/pinned.py @@ -3,13 +3,16 @@ from __future__ import annotations import inspect +import logging import os import re from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from datetime import datetime, timezone from types import MappingProxyType -from typing import Any, Protocol, TypeVar +from typing import Any, Protocol + +from pydantic import ValidationError from ...covenant import ( ArgumentNormalizationError, @@ -20,8 +23,18 @@ from ...event_store import AppendedEvent, EventStreamConflict from ...retrieval import RetrievalQuery from ...workspace import Workspace -from .errors import ErrorCode -from .models import ApiResponse, CapabilityState, RecordSummary, RetrievalData +from .errors import ( + DATABASE_IN_USE_RETRY_AFTER_MS, + MIGRATION_REQUIRED_MESSAGE, + ErrorCode, +) +from .models import ( + ApiResponse, + ApiWarning, + CapabilityState, + RecordSummary, + RetrievalData, +) from .responses import ResponseContext, ResponseFactory from .tasks import ( durable_task_execution_var, @@ -78,36 +91,37 @@ def __init__(self, code: str) -> None: super().__init__(code) -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? bool: - if isinstance(value, str): - return ( - _WINDOWS_ABSOLUTE_PATH.search(value) is not None - or _POSIX_ABSOLUTE_PATH.search(value) is not None - ) - if isinstance(value, Mapping): - return any( - _contains_raw_path(key) or _contains_raw_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(_contains_raw_path(item) for item in value) - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - return _contains_raw_path(model_dump(mode="python")) - return False - +def _token_not_issued_warning(error: ArgumentNormalizationError) -> ApiWarning: + """Say why a description-only preflight carries no token. -T = TypeVar("T") + Supplied values are never echoed: only pydantic error types and the + locations of the failing arguments. A location part is echoed only when + it is an int index or identifier-shaped (which may include a key the + caller sent); anything else, such as a path-like key, becomes ``?``. + """ - -def _path_safe_success(response: ResponseContext, data: T) -> ApiResponse[T]: - if _contains_raw_path(data): - return response.internal_error() - return response.success(data) + problems: list[str] = [] + cause = error.__cause__ + if isinstance(cause, ValidationError): + for item in cause.errors(): + location = ".".join( + str(part) + if isinstance(part, int) or _FIELD_NAME.match(str(part)) + else "?" + for part in item["loc"] + ) + problems.append(f"{location or 'target_arguments'} ({item['type']})") + detail = "; ".join(problems)[:360] or "they do not match the target tool schema" + return ApiWarning( + code="PREFLIGHT_TOKEN_NOT_ISSUED", + message=( + f"Target arguments did not validate: {detail}. " + "No preflight token issued; fix them and call memory_preflight again." + ), + ) _EXPECTED_SERVICE_ERRORS = MappingProxyType( @@ -147,6 +161,11 @@ def _path_safe_success(response: ResponseContext, data: T) -> ApiResponse[T]: "The active v7 workspace is unavailable.", True, ), + "MIGRATION_REQUIRED": ( + ErrorCode.MIGRATION_REQUIRED, + MIGRATION_REQUIRED_MESSAGE, + False, + ), "FEDERATION_UNAVAILABLE": ( ErrorCode.CAPABILITY_DISABLED, "Linked-workspace retrieval is unavailable.", @@ -192,7 +211,23 @@ def _expected_service_failure( if mapped is None: return None code, message, retryable = mapped - return response.failure(code, message, retryable=retryable) + if error_code == "ACTIVE_V7_UNAVAILABLE": + # Opaque on the wire like INTERNAL_ERROR, so keep the cause findable. + logging.getLogger(__name__).warning( + "v7 workspace unavailable correlation_id=%s", + response.request_id, + exc_info=error, + ) + return response.failure( + code, + message, + retryable=retryable, + retry_after_ms=( + DATABASE_IN_USE_RETRY_AFTER_MS + if code is ErrorCode.DATABASE_IN_USE + else None + ), + ) def _utc_now() -> datetime: @@ -437,7 +472,7 @@ async def session_brief( data = SessionBriefData.model_validate(assembled) if data.workspace_id != request.workspace_id: raise ValueError("briefing service returned a mismatched workspace") - result = _path_safe_success(response, data) + result = response.success(data) if result.ok: self._dependencies.covenant_gate.record_briefing(scope) return result @@ -507,13 +542,14 @@ async def memory_preflight( **request.target_arguments, } target_is_complete = True + draft_warning: ApiWarning | None = None try: normalized_arguments = self._dependencies.argument_normalizer( request.target_tool, target_call_arguments, scope.canonical_workspace, ) - except ArgumentNormalizationError: + except ArgumentNormalizationError as normalization_error: if request.description is None: return response.failure( ErrorCode.INVALID_ARGUMENT, @@ -524,6 +560,7 @@ async def memory_preflight( # can never become capability material until full normalization # succeeds. target_is_complete = False + draft_warning = _token_not_issued_warning(normalization_error) normalized_arguments = dict(request.target_arguments) try: guidance_value = self._dependencies.preflight_service.guidance( @@ -538,17 +575,15 @@ async def memory_preflight( else guidance_value ) guidance = PreflightGuidance.model_validate(guidance_result) - if _contains_raw_path(guidance): - return response.internal_error() if not target_is_complete: - return _path_safe_success( - response, + return response.success( PreflightData( guidance=guidance, preflight_token=None, target_tool=request.target_tool, expires_at=None, ), + warnings=[draft_warning] if draft_warning is not None else (), ) token = self._dependencies.covenant_gate.issue_preflight( scope, @@ -563,7 +598,7 @@ async def memory_preflight( target_tool=request.target_tool, expires_at=expires_at, ) - return _path_safe_success(response, data) + return response.success(data) except Exception as exc: return _expected_service_failure(response, exc) or response.internal_error( exc @@ -744,8 +779,7 @@ async def authorize_linked(selected: Workspace) -> None: if inspect.isawaitable(result_value) else result_value ) - return _path_safe_success( - response, + return response.success( RetrievalData.model_validate(result), ) except Exception as exc: @@ -797,6 +831,14 @@ async def memory_store( if failure is not None: return failure assert scope is not None + preflight_remedy = { + "workspace_id": request.workspace_id, + "target_tool": "memory_store", + "target_arguments": request.model_dump( + mode="json", + exclude={"workspace_id", "preflight_token"}, + ), + } violation = self._dependencies.covenant_gate.authorize( "memory_store", request.model_dump(), @@ -823,14 +865,7 @@ async def memory_store( ErrorCode.TOKEN_LEGACY_UNSUPPORTED, }: remedy_tool = "memory_preflight" - remedy_arguments = { - "workspace_id": request.workspace_id, - "target_tool": "memory_store", - "target_arguments": request.model_dump( - mode="json", - exclude={"workspace_id", "preflight_token"}, - ), - } + remedy_arguments = preflight_remedy return response.failure( stable_code, "The preflight capability was rejected.", @@ -866,7 +901,7 @@ async def memory_store( stream_version=stored.event.stream_version, idempotent_replay=stored.idempotent_replay, ) - return _path_safe_success(response, data) + return response.success(data) except IdempotencyConflict: return response.failure( ErrorCode.IDEMPOTENCY_CONFLICT, @@ -879,6 +914,18 @@ async def memory_store( retryable=True, ) except Exception as exc: + if getattr(exc, "code", None) == ErrorCode.DATABASE_IN_USE.value: + # authorize() already spent the token, so a retry needs a new + # one; the idempotency key keeps the retry replay-safe. + return response.failure( + ErrorCode.DATABASE_IN_USE, + "The workspace database is currently in use. Request a new " + "preflight token, then retry with the same idempotency_key.", + retryable=True, + retry_after_ms=DATABASE_IN_USE_RETRY_AFTER_MS, + remedy_tool="memory_preflight", + remedy_arguments=preflight_remedy, + ) return _expected_service_failure(response, exc) or response.internal_error( exc ) @@ -961,7 +1008,7 @@ async def memory_record_outcome( worked=recorded.worked, idempotent_replay=recorded.idempotent_replay, ) - return _path_safe_success(response, data) + return response.success(data) except IdempotencyConflict: return response.failure( ErrorCode.IDEMPOTENCY_CONFLICT, @@ -1029,7 +1076,7 @@ async def system_health( return failure if not request.include_components and data.capability_states: data = data.model_copy(update={"capability_states": []}) - return _path_safe_success(response, data) + return response.success(data) except Exception as exc: return _expected_service_failure(response, exc) or response.internal_error( exc diff --git a/daem0nmcp/api/v7/production.py b/daem0nmcp/api/v7/production.py index 498cd4a..5285ccb 100644 --- a/daem0nmcp/api/v7/production.py +++ b/daem0nmcp/api/v7/production.py @@ -10,7 +10,7 @@ import os import re import secrets -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping from contextlib import asynccontextmanager, suppress from dataclasses import dataclass from datetime import datetime, timezone @@ -225,34 +225,46 @@ async def aclose(self) -> None: await await_projection_job_drains(tuple(self._paths)) +# Only entry points the profile's own extra installs; a module from another +# profile would fail here and degrade a working one. +_NATIVE_PROFILE_MODULES = { + "local": ("qdrant_client",), + "models-local": ("sentence_transformers", "onnx", "onnxruntime"), + "graph": ("networkx", "igraph", "leidenalg"), +} + + class _OptionalNativeRuntimeLifecycle: """Initialize enabled native provider entry points before worker threads.""" - def __init__(self, capability_statuses: Mapping[str, str]) -> None: - self._local_enabled = capability_statuses.get("local") == "ready" - self._models_enabled = capability_statuses.get("models-local") == "ready" - self._graph_enabled = capability_statuses.get("graph") == "ready" + def __init__(self, capability_statuses: MutableMapping[str, str]) -> None: + self._statuses = capability_statuses + self.failures: dict[str, str] = {} def start(self) -> None: - modules = [] - if self._local_enabled: - modules.append("qdrant_client") - if self._models_enabled: - modules.extend(("sentence_transformers", "onnx", "onnxruntime")) - if self._graph_enabled: - modules.extend(("numpy", "networkx", "igraph", "leidenalg")) - for module in modules: - # Import the public model/runtime entry points on the main thread. - # This initializes NumPy/SciPy/native DLL dependencies before a - # dense query can race another provider's first worker creation. - try: - importlib.import_module(module) - except Exception: - # Independent providers still get initialized if one is broken. - # Their operation boundary reports stable degraded diagnostics. - logging.getLogger(__name__).warning( - "Optional native provider runtime preload is unavailable" - ) + for profile, modules in _NATIVE_PROFILE_MODULES.items(): + if self._statuses.get(profile) != "ready": + continue + for module in modules: + # Import the public model/runtime entry points on the main + # thread. This initializes NumPy/SciPy/native DLL dependencies + # before a dense query can race another provider's first worker. + try: + importlib.import_module(module) + except Exception as error: + # Installed metadata said ready, but the package cannot run + # here. Report that once instead of per tool call. + self._statuses[profile] = "degraded" + self.failures[profile] = ( + f"importing {module} raised {type(error).__name__}" + ) + logging.getLogger(__name__).warning( + "Optional %s runtime is unavailable: importing %s raised %s", + profile, + module, + type(error).__name__, + ) + break def _loopback_host(host: str) -> bool: @@ -304,29 +316,56 @@ def _task_configuration( ) +def _remediation_text(name: str, remediation: Mapping[str, Any]) -> str: + """Render the registry's structured remediation as one actionable line.""" + + parts = [ + str(remediation[key]) + for key in ("message", "command", "environment") + if remediation.get(key) + ] + missing = remediation.get("missing") + if isinstance(missing, list) and missing: + parts.append("Missing: " + ", ".join(str(item) for item in missing[:8]) + ".") + return " ".join(parts) or f"Review the {name} capability profile." + + def _capability_states( environ: Mapping[str, str], + native_failures: Mapping[str, str] = MappingProxyType({}), ) -> tuple[CapabilityState, ...]: values: list[CapabilityState] = [] for name, capability in CapabilityRegistry(environ=environ).all().items(): status = str(capability["status"]) - if status == "ready": + failure = native_failures.get(name) + if status == "ready" and failure is None: values.append( CapabilityState.model_validate({"name": name, "status": "ready"}) ) continue + if failure is not None: + status = "degraded" reason = { "disabled": "CAPABILITY_DISABLED", "degraded": "CAPABILITY_DEGRADED", "failed": "CAPABILITY_CONFIGURATION_INVALID", }[status] + remediation = capability.get("remediation") + text = ( + f"The {name} profile is installed but cannot run here: {failure}. " + "Reinstall or repair it." + if failure is not None + else _remediation_text( + name, remediation if isinstance(remediation, Mapping) else {} + ) + ) values.append( CapabilityState.model_validate( { "name": name, "status": status, "reason_code": reason, - "remediation": f"Review the {name} capability profile.", + "remediation": text, } ) ) @@ -336,6 +375,10 @@ def _capability_states( def _active_database(workspace: Workspace): storage = resolve_workspace_storage(workspace) active = resolve_active_database(storage) + if active.format_version == 6: + # The store is intact but still v6; retrying never helps, so say so + # and name the offline command that migrates it. + raise ProductionConfigurationError("MIGRATION_REQUIRED") if active.format_version != 7: raise ProductionConfigurationError("ACTIVE_V7_UNAVAILABLE") return active @@ -460,6 +503,8 @@ async def read( ) -> dict[str, object]: try: _active_database(workspace) + except ProductionConfigurationError as exc: + raise RuntimeServiceError(exc.code) from None except Exception: raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None warning_limit = request.warning_limit @@ -605,6 +650,7 @@ async def read( failure_limit=20, rule_limit=50, active_context_limit=1, + include_git_changes=False, ) candidate_records = [ *_public_items(snapshot.failures, RecordSummary), @@ -709,6 +755,10 @@ def _assemble( name: str(capability["status"]) for name, capability in CapabilityRegistry(environ=env).all().items() } + # Preload before anything snapshots the statuses, so an installed but + # broken extra degrades everywhere instead of only failing per call. + native_runtimes = _OptionalNativeRuntimeLifecycle(capability_statuses) + native_runtimes.start() normalizer = build_argument_normalizer() registry = WorkspaceRegistry.from_settings(loaded_settings) workspaces = {registry.default.workspace_id: registry.default} @@ -837,7 +887,7 @@ def schedule_capture_projection(path: Path) -> None: else "jwt" ), task_support=task_state, - capability_states=_capability_states(env), + capability_states=_capability_states(env, native_runtimes.failures), storage_resolver=storage_resolver, dreaming_provider=dreaming.health, runtime_diagnostics_provider=runtime_health.inspect, @@ -983,7 +1033,6 @@ def authorize_bridge_workspace( edit_bridge_service=edit_bridge_service, services=( WorkspaceBootstrapLifecycle(tuple(workspaces.values())), - _OptionalNativeRuntimeLifecycle(capability_statuses), writer, recall, runtime_health, diff --git a/daem0nmcp/api/v7/record_operations.py b/daem0nmcp/api/v7/record_operations.py index c5d1276..49939f7 100644 --- a/daem0nmcp/api/v7/record_operations.py +++ b/daem0nmcp/api/v7/record_operations.py @@ -46,11 +46,13 @@ from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET from .models import ( + MIGRATED_EMPTY_CONTENT, EvidenceRef, MutationReceipt, Page, RecordSummary, parse_wire_datetime, + stored_relative_path, ) from .pinned import IdempotencyConflict from .runtime_services import WorkspaceStorageResolver @@ -307,8 +309,10 @@ def _load_record( ).fetchall() if not rows: raise RecordOperationError("NOT_FOUND") - if len(rows) != 1 or rows[0]["file_path"] is not None: + if len(rows) != 1: raise RecordOperationError("CAPABILITY_DEGRADED") + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; it is + # legacy provenance, never emitted, so it must not deny the read. return rows[0] @@ -325,16 +329,16 @@ def _summary( *, include_metadata: bool = True, ) -> RecordSummary: + # v6 accepted empty content; the bounded summary renders it as the + # migration's marker rather than denying the whole read. content = row["content"] - if not isinstance(content, str) or not content: + if not isinstance(content, str): raise RecordOperationError("CAPABILITY_DEGRADED") tags = _parse_json(row["tags_json"], list) + # A future valid-time may be recorded now; ``RecordSummary`` exposes the + # transaction-time creation while the event retains the exact valid time. created_at = _datetime_from_us(row["created_at_us"]) updated_at = _datetime_from_us(row["updated_at_us"]) - # A future valid-time may be recorded now. The bounded summary exposes - # its transaction-time creation while the event retains exact valid-time. - if created_at > updated_at: - created_at = updated_at try: return RecordSummary.model_validate( { @@ -342,9 +346,9 @@ def _summary( "record_type": str(row["record_type"]), "excerpt": content[:4000], "tags": tags if include_metadata else [], - "relative_file_path": None - if not include_metadata or row["file_path_relative"] is None - else str(row["file_path_relative"]), + "relative_file_path": stored_relative_path(row["file_path_relative"]) + if include_metadata + else None, "current_status": _record_status(row), "content_hash": str(row["content_hash"]), "created_at": created_at, @@ -366,7 +370,7 @@ def _record_state(row: sqlite3.Row) -> dict[str, Any]: "context": _parse_json(row["context_json"], dict), "tags": _parse_json(row["tags_json"], list), "file_path": None, - "file_path_relative": row["file_path_relative"], + "file_path_relative": stored_relative_path(row["file_path_relative"]), "keywords": row["keywords"], "is_permanent": bool(row["is_permanent"]), "pinned": bool(row["pinned"]), @@ -1070,7 +1074,9 @@ def _text_search_sync( or bool(row["archived"]) ): raise RecordOperationError("LEXICAL_UNAVAILABLE") - excerpt = str(row["content"])[:8000] + # v6 accepted empty content; the bounded excerpt shows + # the migration marker rather than failing the search. + excerpt = str(row["content"])[:8000] or MIGRATED_EMPTY_CONTENT evidence = candidate.evidence hits.append( TextSearchHit( diff --git a/daem0nmcp/api/v7/relationship_operations.py b/daem0nmcp/api/v7/relationship_operations.py index 08ff11b..c6c8b92 100644 --- a/daem0nmcp/api/v7/relationship_operations.py +++ b/daem0nmcp/api/v7/relationship_operations.py @@ -353,7 +353,9 @@ async def _run_read( def _record_summary(row: sqlite3.Row) -> RecordSummary: try: content = row["content"] - if not isinstance(content, str) or not content or row["file_path"] is not None: + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote; it + # is legacy provenance, never emitted, so it must not deny the read. + if not isinstance(content, str) or not content: raise ValueError tags = json.loads(str(row["tags_json"])) if not isinstance(tags, list): diff --git a/daem0nmcp/api/v7/resource_repository.py b/daem0nmcp/api/v7/resource_repository.py index c1c7a07..4626698 100644 --- a/daem0nmcp/api/v7/resource_repository.py +++ b/daem0nmcp/api/v7/resource_repository.py @@ -27,10 +27,12 @@ from typing import TypeVar, cast from ...bounded_workers import BoundedWorkerPool +from ...retrieval.lexical_config import RETRIEVAL_PROJECTION_NAMES from ...schema_version import CURRENT_SCHEMA_VERSION from ...storage_activation import DatabaseFileLock, ResolvedActiveDatabase from ...workspace import Workspace, normalize_resolved_path -from .models import RecordSummary +from .errors import is_database_busy +from .models import RecordSummary, stored_relative_path from .public_ids import PublicObjectIdRepository from .resources import ( RESOURCE_FETCH_LIMIT, @@ -182,9 +184,9 @@ def close(self) -> None: _MAX_GIT_OUTPUT_BYTES = 1_048_576 _MAX_GIT_CHANGES = 200 _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") -_PROJECTION_NAMES = frozenset( - {"lexical", "dense", "graph", "temporal", "procedure", "outcome", "code"} -) +_PROJECTION_NAMES = RETRIEVAL_PROJECTION_NAMES +_PROJECTION_NAME_PLACEHOLDERS = ",".join("?" * len(_PROJECTION_NAMES)) +_PROJECTION_NAME_PARAMETERS = tuple(sorted(_PROJECTION_NAMES)) _REQUIRED_TABLES = frozenset( { "active_context", @@ -209,8 +211,9 @@ class ResourceRepositoryError(RuntimeError): code = "RESOURCE_REPOSITORY_UNAVAILABLE" - def __init__(self) -> None: - super().__init__(self.code) + def __init__(self, code: str = "RESOURCE_REPOSITORY_UNAVAILABLE") -> None: + self.code = code + super().__init__(code) ActiveDatabaseResolver = Callable[[Workspace], ResolvedActiveDatabase] @@ -336,32 +339,6 @@ def _string_list(value: object) -> list[str]: return decoded -def _workspace_root_tokens(workspace: Workspace) -> tuple[str, ...]: - root = str(workspace.root) - variants = {root.casefold(), root.replace("\\", "/").casefold()} - return tuple(value for value in variants if value) - - -def _reject_canonical_root(value: object, workspace: Workspace) -> None: - tokens = _workspace_root_tokens(workspace) - - def walk(item: object) -> None: - if isinstance(item, str): - normalized = item.casefold() - portable = item.replace("\\", "/").casefold() - if any(token in normalized or token in portable for token in tokens): - raise ValueError("public value contains a canonical workspace root") - elif isinstance(item, Mapping): - for key, child in item.items(): - walk(key) - walk(child) - elif isinstance(item, (list, tuple)): - for child in item: - walk(child) - - walk(value) - - def _validated_now(clock: Callable[[], datetime]) -> datetime: value = clock() if ( @@ -519,8 +496,13 @@ async def read_briefing_snapshot( failure_limit: int, rule_limit: int = 50, active_context_limit: int = 50, + include_git_changes: bool = True, ) -> ResourceRepositorySnapshot: - """Read all briefing resources without a pointer-generation gap.""" + """Read all briefing resources without a pointer-generation gap. + + Git changes cost two ``git`` subprocesses per read (plus a system-wide + thread walk on Windows); callers that never show them skip them. + """ if not isinstance(workspace, Workspace): raise ValueError("workspace must be a registered Workspace") @@ -561,6 +543,7 @@ async def read_briefing_snapshot( failure_request=failure_request, rule_request=rule_request, active_request=active_request, + include_git_changes=include_git_changes is True, ) ) @@ -572,6 +555,7 @@ def _read_briefing_snapshot_sync( failure_request: ResourceReadRequest | None, rule_request: ResourceReadRequest, active_request: ResourceReadRequest, + include_git_changes: bool = True, ) -> ResourceRepositorySnapshot: connection, storage_lock = self._open_connection(workspace) try: @@ -643,7 +627,9 @@ def _read_briefing_snapshot_sync( rules=rules, active_context=active_context, decisions=decisions, - git_changes=self._read_git_changes_sync(workspace), + git_changes=( + self._read_git_changes_sync(workspace) if include_git_changes else [] + ), projection_freshness=projection_freshness, workspace_statistics=workspace_statistics, stale_projection_count=stale_projection_count, @@ -793,12 +779,17 @@ def _read_projection_freshness_sync( workspace: Workspace, connection: sqlite3.Connection, ) -> tuple[list[object], int]: + # ``projection_manifests`` also holds the migration's table snapshots + # (``memory_records`` and friends), which no retrieval provider serves. + # Select the retrieval names instead of refusing the rest: a migrated + # workspace must brief without a data change. rows = connection.execute( "SELECT projection_name,generation,source_event_root_hash," "COALESCE(activated_at_us,completed_at_us,started_at_us) AS built_at_us " "FROM projection_manifests WHERE workspace_id=? AND status='active' " + f"AND projection_name IN ({_PROJECTION_NAME_PLACEHOLDERS}) " "ORDER BY projection_name", - (workspace.workspace_id,), + (workspace.workspace_id, *_PROJECTION_NAME_PARAMETERS), ).fetchall() values: list[object] = [] for row in rows: @@ -806,7 +797,6 @@ def _read_projection_freshness_sync( root_hash = row["source_event_root_hash"] if ( not isinstance(projection, str) - or projection not in _PROJECTION_NAMES or not isinstance(root_hash, str) or _SHA256_RE.fullmatch(root_hash) is None ): @@ -828,13 +818,12 @@ def _read_projection_freshness_sync( "json_extract(details_json,'$.rebuild_required_event_id') IS NOT NULL " "THEN 1 ELSE 0 END) AS marked_stale " "FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({_PROJECTION_NAME_PLACEHOLDERS}) " "GROUP BY projection_name", - (workspace.workspace_id,), + (workspace.workspace_id, *_PROJECTION_NAME_PARAMETERS), ).fetchall() stale = 0 for row in state_rows: - if row["projection_name"] not in _PROJECTION_NAMES: - raise ValueError("projection name is invalid") if ( _plain_int(row["has_active"]) == 0 or _plain_int(row["marked_stale"]) == 1 @@ -949,6 +938,7 @@ def _workspace_statistics_sync( async def _run(self, operation: Callable[[], T]) -> T: worker = asyncio.create_task(self._worker_pool.run(operation)) cancellation: asyncio.CancelledError | None = None + timed_out = False try: return cast( T, @@ -958,7 +948,7 @@ async def _run(self, operation: Callable[[], T]) -> T: ), ) except asyncio.TimeoutError: - pass + timed_out = True except asyncio.CancelledError as exc: cancellation = exc except Exception: @@ -978,12 +968,16 @@ async def _run(self, operation: Callable[[], T]) -> T: continue except Exception: break - if worker.done(): - with suppress(Exception): - worker.result() + failure: BaseException | None = None + if worker.done() and not worker.cancelled(): + failure = worker.exception() if cancellation is not None: raise cancellation - raise ResourceRepositoryError() + # A read that overran its deadline but did not itself fail was slowed + # by contention (lock waits, a saturated host), so it is retryable. + if is_database_busy(failure) or (timed_out and failure is None): + raise ResourceRepositoryError("DATABASE_IN_USE") from failure + raise ResourceRepositoryError() from failure def _open_connection( self, @@ -1123,16 +1117,16 @@ def _record_summary( if not isinstance(record_type, str) or record_type not in _PUBLIC_RECORD_TYPES: raise ValueError("record type is not public") content = row["content"] - if not isinstance(content, str) or not content: + # v6 accepted empty content; ``RecordSummary`` renders it as the + # migration's marker rather than denying the whole read. + if not isinstance(content, str): raise ValueError("record content is invalid") tags = _string_list(row["tags_json"]) archived = _flag(row["archived"]) deleted = row["deleted_at_us"] is not None if deleted: _datetime_from_us(row["deleted_at_us"]) - relative_path = row["file_path_relative"] - if relative_path is not None and not isinstance(relative_path, str): - raise ValueError("relative file path is invalid") + relative_path = stored_relative_path(row["file_path_relative"]) status = "archived" if archived else "invalidated" if deleted else "current" public_value = { "record_id": row["record_id"], @@ -1145,7 +1139,6 @@ def _record_summary( "created_at": _datetime_from_us(row["created_at_us"]), "updated_at": _datetime_from_us(row["updated_at_us"]), } - _reject_canonical_root(public_value, workspace) return RecordSummary.model_validate(public_value), deleted def _read_rules_sync( @@ -1191,7 +1184,6 @@ def _read_rules_sync( "enabled": _flag(row["enabled"]), "created_at": _datetime_from_us(row["created_at_us"]), } - _reject_canonical_root(public_value, workspace) values.append(RuleView.model_validate(public_value)) return values finally: @@ -1417,7 +1409,6 @@ def _read_active_context_sync( "added_at": added_at, "expires_at": expires_at, } - _reject_canonical_root(public_value, workspace) item = ActiveContextItem.model_validate(public_value) ordered_values.append( ( diff --git a/daem0nmcp/api/v7/resources.py b/daem0nmcp/api/v7/resources.py index e19f4c5..b488586 100644 --- a/daem0nmcp/api/v7/resources.py +++ b/daem0nmcp/api/v7/resources.py @@ -23,6 +23,7 @@ AwareDateTime, RecordSummary, RuleId, + UserText, UtcDateTime, WireModel, WorkspaceId, @@ -49,8 +50,9 @@ _UTC_DATETIME_ADAPTER = TypeAdapter(UtcDateTime) _ACCESS_FAILURE = object() +# Rule text is user-authored and may mention paths. BoundedText = Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] @@ -91,7 +93,7 @@ class ActiveContextItem(WireModel): priority: Annotated[int, Field(strict=True, ge=-100, le=100)] reason: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None diff --git a/daem0nmcp/api/v7/responses.py b/daem0nmcp/api/v7/responses.py index fa51258..b913108 100644 --- a/daem0nmcp/api/v7/responses.py +++ b/daem0nmcp/api/v7/responses.py @@ -2,12 +2,18 @@ from __future__ import annotations +import logging +import re import secrets +import sys +import traceback from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, TypeVar +from pydantic import ValidationError + from .errors import INTERNAL_ERROR_MESSAGE, ErrorCode from .models import ( ApiError, @@ -22,6 +28,52 @@ T = TypeVar("T") +_LOGGER = logging.getLogger(__name__) +_LOCATION_PART = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") + + +def _exception_chain(error: BaseException) -> list[BaseException]: + """Return *error* and its causes, root cause first, as tracebacks do.""" + + chain: list[BaseException] = [] + link: BaseException | None = error + while link is not None and all(link is not seen for seen in chain): + chain.append(link) + link = link.__cause__ or ( + None if link.__suppress_context__ else link.__context__ + ) + return chain[::-1] + + +def _without_input_values(chain: list[BaseException]) -> str: + """Render a traceback whose pydantic errors omit the offending values. + + ``str(ValidationError)`` quotes a slice of each invalid input, which for + service output can be stored memory content or a token. + """ + + rendered = [] + for link in chain: + if isinstance(link, ValidationError): + detail = "; ".join( + ".".join( + str(part) + if isinstance(part, int) or _LOCATION_PART.match(str(part)) + else "?" + for part in item["loc"] + ) + + f" ({item['type']})" + for item in link.errors(include_url=False) + ) + else: + detail = str(link) + rendered.append( + "Traceback (most recent call last):\n" + + "".join(traceback.format_tb(link.__traceback__)) + + f"{type(link).__module__}.{type(link).__qualname__}: {detail}" + ) + return "\n\nThe above exception led to:\n\n".join(rendered) + def _utc_now() -> datetime: return datetime.now(timezone.utc) @@ -118,9 +170,28 @@ def failure( ) def internal_error(self, error: BaseException | None = None) -> ApiResponse[Any]: - """Return the one deliberately opaque internal failure envelope.""" - - del error + """Return the one deliberately opaque internal failure envelope. + + The cause is logged (stderr by default, never the MCP wire) under the + correlation ID the caller receives, so the owner can find it. Logs + may still hold short exception messages such as local paths; redact + them before sharing. Pydantic input values are never logged. + """ + + cause = error if error is not None else sys.exc_info()[1] + chain = [] if cause is None else _exception_chain(cause) + if any(isinstance(link, ValidationError) for link in chain): + _LOGGER.error( + "v7 internal error correlation_id=%s\n%s", + self.request_id, + _without_input_values(chain), + ) + else: + _LOGGER.error( + "v7 internal error correlation_id=%s", + self.request_id, + exc_info=cause, + ) return self.failure( ErrorCode.INTERNAL_ERROR, INTERNAL_ERROR_MESSAGE, diff --git a/daem0nmcp/api/v7/runtime_services.py b/daem0nmcp/api/v7/runtime_services.py index aada0f8..68cb407 100644 --- a/daem0nmcp/api/v7/runtime_services.py +++ b/daem0nmcp/api/v7/runtime_services.py @@ -11,7 +11,6 @@ import inspect import json import os -import re import sqlite3 import threading import time @@ -49,6 +48,7 @@ normalize_resolved_path, resolve_derived_path, ) +from .errors import is_database_busy from .federated_retrieval import ( FederatedCandidate, FederatedRetrievalError, @@ -64,6 +64,7 @@ RecordSummary, RetrievalData, TokenUsage, + stored_relative_path, ) from .models import ( EvidenceItem as PublicEvidenceItem, @@ -89,8 +90,6 @@ SessionBriefInput, ) -_WINDOWS_ABSOLUTE_PATH = re.compile(r"(? bool: - if isinstance(value, str): - return bool( - _WINDOWS_ABSOLUTE_PATH.search(value) or _POSIX_ABSOLUTE_PATH.search(value) - ) - if isinstance(value, Mapping): - return any( - _contains_raw_path(key) or _contains_raw_path(item) - for key, item in value.items() - ) - if isinstance(value, (list, tuple, set, frozenset)): - return any(_contains_raw_path(item) for item in value) - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - return _contains_raw_path(model_dump(mode="python")) - return False - - def _validated_workspace(workspace: Workspace) -> Workspace: if not isinstance(workspace, Workspace): raise RuntimeServiceError("INVALID_WORKSPACE") try: canonical = normalize_resolved_path(workspace.root.resolve(strict=True)) registered = WorkspaceRegistry([canonical], default_root=canonical).default - except (OSError, RuntimeError, TypeError, ValueError): - raise RuntimeServiceError("INVALID_WORKSPACE") from None + except (OSError, RuntimeError, TypeError, ValueError) as exc: + raise RuntimeServiceError("INVALID_WORKSPACE") from exc if ( registered.workspace_id != workspace.workspace_id or os.path.normcase(str(registered.root)) != os.path.normcase(str(canonical)) @@ -167,8 +148,8 @@ def resolve_workspace_storage(workspace: Workspace) -> Path: ".daem0nmcp", "storage", ) - except Exception: - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc if not storage.is_dir() or storage.is_symlink(): raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") return storage @@ -194,9 +175,13 @@ def locked_current( except RuntimeServiceError: lock.release() raise - except Exception: + except Exception as exc: lock.release() - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + # DatabaseInUseError: another process holds the storage lock + # exclusively (fresh bootstrap, migration) - transient, retryable. + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc try: yield active finally: @@ -208,6 +193,9 @@ def locked_active( workspace: Workspace, ) -> Iterator[ResolvedActiveDatabase]: with self.locked_current(workspace) as active: + if active.format_version == 6: + # An intact but un-migrated v6 store: retrying never helps. + raise RuntimeServiceError("MIGRATION_REQUIRED") if ( active.pointer is None or active.format_version != _FORMAT_VERSION @@ -264,10 +252,12 @@ def _open_database(path: Path, *, writable: bool) -> sqlite3.Connection: with suppress(NameError, sqlite3.Error): connection.close() raise - except Exception: + except Exception as exc: with suppress(NameError, sqlite3.Error): connection.close() - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc def _inspect_database_health( @@ -319,8 +309,8 @@ def _inspect_database_health( # structurally incomplete storage. Full-page corruption certification # remains the explicit offline verify-v7 operation. return schema_version, tables - except Exception: - raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("ACTIVE_V7_UNAVAILABLE") from exc finally: if connection is not None: with suppress(sqlite3.Error): @@ -335,8 +325,8 @@ def _datetime_us(value: datetime) -> int: epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) delta = utc - epoch result = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds - except (OverflowError, ValueError): - raise RuntimeServiceError("INVALID_TIMESTAMP") from None + except (OverflowError, ValueError) as exc: + raise RuntimeServiceError("INVALID_TIMESTAMP") from exc if not -(2**63) <= result <= 2**63 - 1: raise RuntimeServiceError("INVALID_TIMESTAMP") return result @@ -347,8 +337,8 @@ def _datetime_from_us(value: object) -> datetime: raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") try: return datetime.fromtimestamp(value / 1_000_000, timezone.utc) - except (OverflowError, OSError, ValueError): - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from None + except (OverflowError, OSError, ValueError) as exc: + raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from exc def _record_status(row: sqlite3.Row, evidence_status: str = "current") -> str: @@ -364,8 +354,8 @@ def _record_status(row: sqlite3.Row, evidence_status: str = "current") -> str: def _parse_json(value: object, expected_type: type, code: str) -> Any: try: parsed = json.loads(str(value)) - except (TypeError, ValueError, RecursionError): - raise RuntimeServiceError(code) from None + except (TypeError, ValueError, RecursionError) as exc: + raise RuntimeServiceError(code) from exc if not isinstance(parsed, expected_type): raise RuntimeServiceError(code) return parsed @@ -384,8 +374,8 @@ def _event_receipt(row: sqlite3.Row) -> AppendedEvent: else str(row["previous_event_hash"]) ), ) - except (KeyError, TypeError, ValueError): - raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from None + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from exc def _idempotency_correlation( @@ -427,8 +417,8 @@ def _existing_idempotent_event( payload = _parse_json(row["payload_json"], dict, "MEMORY_EVENT_INTEGRITY_FAILED") try: canonical = canonical_json_bytes(payload).decode("utf-8") - except Exception: - raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("MEMORY_EVENT_INTEGRITY_FAILED") from exc if canonical != str(row["payload_json"]) or sha256_json(payload) != str( row["payload_hash"] ): @@ -462,8 +452,9 @@ def _load_record_row( ).fetchall() if len(rows) != 1: raise RuntimeServiceError("NOT_FOUND") - if rows[0]["file_path"] is not None: - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") + # A migrated v6 row keeps the host-absolute ``file_path`` v6 wrote. It is + # never emitted (``_record_summary`` and ``_record_state`` drop it), so it + # is legacy provenance, not corruption. return rows[0] @@ -473,8 +464,10 @@ def _record_summary( evidence_status: str = "current", ) -> RecordSummary: tags = _parse_json(row["tags_json"], list, "MEMORY_RECORD_INTEGRITY_FAILED") + # v6 accepted empty content; the bounded summary renders it as the + # migration's marker rather than denying the whole read. content = row["content"] - if not isinstance(content, str) or not content: + if not isinstance(content, str): raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") try: return RecordSummary.model_validate( @@ -483,9 +476,7 @@ def _record_summary( "record_type": str(row["record_type"]), "excerpt": content[:4000], "tags": tags, - "relative_file_path": None - if row["file_path_relative"] is None - else str(row["file_path_relative"]), + "relative_file_path": stored_relative_path(row["file_path_relative"]), "current_status": _record_status(row, evidence_status), "content_hash": str(row["content_hash"]), "created_at": _datetime_from_us(row["created_at_us"]), @@ -494,8 +485,8 @@ def _record_summary( ) except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("MEMORY_RECORD_INTEGRITY_FAILED") from exc def _record_state(row: sqlite3.Row) -> dict[str, Any]: @@ -509,7 +500,7 @@ def _record_state(row: sqlite3.Row) -> dict[str, Any]: "context": context, "tags": tags, "file_path": None, - "file_path_relative": row["file_path_relative"], + "file_path_relative": stored_relative_path(row["file_path_relative"]), "keywords": row["keywords"], "is_permanent": bool(row["is_permanent"]), "pinned": bool(row["pinned"]), @@ -585,6 +576,8 @@ async def _run_mutation( worker = asyncio.create_task(self._workers.run(lambda: operation(cancelled))) try: return await asyncio.shield(worker) + except BoundedWorkerBusyError as exc: + raise RuntimeServiceError("DATABASE_IN_USE") from exc except asyncio.CancelledError as cancellation: cancelled.set() try: @@ -598,8 +591,8 @@ async def _run_mutation( def _now_us(self) -> int: try: value = self._clock() - except Exception: - raise RuntimeServiceError("CLOCK_UNAVAILABLE") from None + except Exception as exc: + raise RuntimeServiceError("CLOCK_UNAVAILABLE") from exc return _datetime_us(value) def _schedule_after_commit(self, path: Path, changed: bool) -> None: @@ -760,10 +753,12 @@ def _store_sync( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("MEMORY_STORE_FAILED") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("MEMORY_STORE_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -880,10 +875,12 @@ def _record_outcome_sync( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("MEMORY_OUTCOME_FAILED") from None + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("MEMORY_OUTCOME_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -949,8 +946,8 @@ def _retrieval_config_fingerprint( "capabilities": dict(sorted(capability_statuses.items())), } ) - except (TypeError, ValueError): - raise RuntimeServiceError("RETRIEVAL_CONFIG_INVALID") from None + except (TypeError, ValueError) as exc: + raise RuntimeServiceError("RETRIEVAL_CONFIG_INVALID") from exc @dataclass(slots=True) @@ -1084,8 +1081,8 @@ async def _run_snapshot(self, operation: Callable[[], Any]) -> Any: worker = asyncio.create_task(self._workers.run(operation)) try: return await asyncio.shield(worker) - except BoundedWorkerBusyError: - raise RuntimeServiceError("RETRIEVAL_UNAVAILABLE") from None + except BoundedWorkerBusyError as exc: + raise RuntimeServiceError("RETRIEVAL_UNAVAILABLE") from exc except asyncio.CancelledError: with suppress(asyncio.CancelledError, Exception): await await_task_terminal(worker) @@ -1132,10 +1129,8 @@ async def retrieve( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("RETRIEVAL_FAILED") from None - if _contains_raw_path(hydrated): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") + except Exception as exc: + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc return hydrated async def _retrieve_source( @@ -1167,8 +1162,8 @@ def _resolve_federated_workspace(self, workspace_id: str) -> Workspace: try: workspace = resolve(workspace_id) validated = _validated_workspace(workspace) - except Exception: - raise RuntimeServiceError("INVALID_WORKSPACE") from None + except Exception as exc: + raise RuntimeServiceError("INVALID_WORKSPACE") from exc if validated.workspace_id != workspace_id: raise RuntimeServiceError("INVALID_WORKSPACE") return validated @@ -1254,14 +1249,12 @@ async def retrieve_one( if inspect.isawaitable(value): await value hydrated = compose_federated_results(dict(pairs), query) - if _contains_raw_path(hydrated): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") finally: guard.release() - except asyncio.TimeoutError: - raise RuntimeServiceError("DEADLINE_EXCEEDED") from None + except asyncio.TimeoutError as exc: + raise RuntimeServiceError("DEADLINE_EXCEEDED") from exc except FederatedRetrievalError as exc: - raise RuntimeServiceError(exc.code) from None + raise RuntimeServiceError(exc.code) from exc except asyncio.CancelledError: raise except RuntimeServiceError: @@ -1272,8 +1265,8 @@ async def retrieve_one( "COMMUNION_REQUIRED", "UNAUTHORIZED_WORKSPACE", }: - raise RuntimeServiceError(authorization_code) from None - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + raise RuntimeServiceError(authorization_code) from exc + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc # `hydrated` is immutable and there is no await between releasing the # read guard above and committing this response to the caller. return hydrated @@ -1328,7 +1321,7 @@ def _hydrate_candidates( "AND event.stream_id=record.record_id LIMIT 2", (query.workspace_id, *key), ).fetchall() - if len(rows) != 1 or rows[0]["file_path"] is not None: + if len(rows) != 1: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") hydrated[key] = rows[0] @@ -1373,10 +1366,10 @@ def _hydrate_candidates( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -1428,8 +1421,8 @@ def _hydrate( dropped=0, ), ) - except Exception: - raise RuntimeServiceError("RETRIEVAL_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("RETRIEVAL_FAILED") from exc if result.context is None: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") @@ -1460,7 +1453,7 @@ def _hydrate( "AND event.stream_id=record.record_id LIMIT 2", (query.workspace_id, *key), ).fetchall() - if len(rows) != 1 or rows[0]["file_path"] is not None: + if len(rows) != 1: raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") hydrated[key] = rows[0] @@ -1541,10 +1534,10 @@ def _hydrate( if connection.in_transaction: connection.rollback() raise - except Exception: + except Exception as exc: if connection.in_transaction: connection.rollback() - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc finally: if connection.in_transaction: connection.rollback() @@ -1565,8 +1558,8 @@ def _public_ref(ref: EvidenceRef, origin_workspace_id: str) -> PublicEvidenceRef relation_path=list(ref.relation_path), provider=provider, ) - except Exception: - raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from None + except Exception as exc: + raise RuntimeServiceError("EVIDENCE_AUTHENTICATION_FAILED") from exc class BriefingReader(Protocol): @@ -1613,19 +1606,17 @@ async def assemble( value = self._reader(workspace, request) if inspect.isawaitable(value): value = await value - if _contains_raw_path(value): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") data = SessionBriefData.model_validate(value) except asyncio.CancelledError: raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("BRIEFING_FAILED") from None + except Exception as exc: + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("BRIEFING_FAILED") from exc if data.workspace_id != workspace.workspace_id: raise RuntimeServiceError("BRIEFING_FAILED") - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data @@ -1678,10 +1669,10 @@ async def guidance( raise except RuntimeServiceError: raise - except Exception: - raise RuntimeServiceError("PREFLIGHT_FAILED") from None - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") + except Exception as exc: + if is_database_busy(exc): + raise RuntimeServiceError("DATABASE_IN_USE") from exc + raise RuntimeServiceError("PREFLIGHT_FAILED") from exc return data @@ -1824,8 +1815,6 @@ def __init__( ) except Exception: raise ValueError("health service configuration is invalid") from None - if _contains_raw_path(self._data): - raise ValueError("health service configuration is unsafe") self._workers = BoundedWorkerPool( max_workers=2, thread_name_prefix="daem0nmcp-v7-health", @@ -1904,8 +1893,6 @@ async def inspect( "runtime_diagnostics": diagnostics, } ) - if _contains_raw_path(data): - raise RuntimeServiceError("UNSAFE_SERVICE_OUTPUT") return data diff --git a/daem0nmcp/api/v7/tools.py b/daem0nmcp/api/v7/tools.py index fb5d7bc..4c48ab9 100644 --- a/daem0nmcp/api/v7/tools.py +++ b/daem0nmcp/api/v7/tools.py @@ -53,19 +53,19 @@ Tag, ToolName, TriggerId, + UniqueTags, + UserJsonObject, + UserText, UtcDateTime, VersionId, WireModel, WorkspaceId, + contains_absolute_filesystem_path, ) from .policy import V7_TOOL_LEVELS, V7ArgumentNormalizer from .registry import PINNED_TOOL_NAMES, ManifestError, ToolSpec from .resources import ActiveContextItem, RuleView -ShortText = Annotated[ - str, - StringConstraints(strict=True, min_length=1, max_length=80), -] NameText = Annotated[ str, StringConstraints(strict=True, min_length=1, max_length=256), @@ -74,17 +74,34 @@ str, StringConstraints(strict=True, min_length=1, max_length=2000), ] -LongText = Annotated[ - str, +# User-authored or user-derived text, which may mention paths (see UserText). +UserShortText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=80), +] +UserNameText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=256), +] +UserMediumText = Annotated[ + UserText, + StringConstraints(strict=True, min_length=1, max_length=2000), +] +UserLongText = Annotated[ + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] -OptionalMediumText = ( - Annotated[ - str, - StringConstraints(strict=True, min_length=1, max_length=2000), - ] - | None -) +OptionalUserMediumText = UserMediumText | None +RecordVisibility = Literal["workspace", "private", "shared"] + + +def _record_visibility(context: dict[str, object]) -> dict[str, object]: + if context.get("visibility", "workspace") not in get_args(RecordVisibility): + raise ValueError("context.visibility must be workspace, private or shared") + return context + + +RecordContext = Annotated[ContextJsonObject, AfterValidator(_record_visibility)] IdempotencyKey = Annotated[ str, StringConstraints( @@ -330,8 +347,8 @@ class DiagnosticSummary(WireModel): class TriggerView(WireModel): trigger_id: TriggerId trigger_type: TriggerType - pattern: MediumText - recall_query: MediumText + pattern: UserMediumText + recall_query: UserMediumText categories: RecordTypeSet | None = None enabled: bool updated_at: UtcDateTime @@ -346,7 +363,7 @@ class OutcomeSummary(WireModel): record_id: RecordId worked: bool outcome_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=4000), ] happened_at: UtcDateTime @@ -378,10 +395,10 @@ class SessionBriefData(WireModel): class PreflightGuidance(WireModel): records: list[RecordSummary] = Field(default_factory=list, max_length=20) rules: list[RuleView] = Field(default_factory=list, max_length=20) - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) class PreflightData(WireModel): @@ -413,7 +430,7 @@ class EditPreflightData(WireModel): class CaptureCandidateView(WireModel): candidate_id: CaptureCandidateId source_kind: Literal["native_edit", "tool_result", "dreaming", "system"] - proposed_record: JsonObject + proposed_record: UserJsonObject provenance: JsonObject created_at: UtcDateTime @@ -521,7 +538,7 @@ class HealthData(WireModel): class TriggerMatch(WireModel): trigger: TriggerView - matched_value: NameText + matched_value: UserNameText records: list[RecordSummary] = Field(default_factory=list, max_length=20) @@ -564,7 +581,7 @@ class HierarchyLayer(WireModel): class CommunitySummary(WireModel): community_id: CommunityId - label: NameText + label: UserNameText level: Annotated[int, Field(ge=0, le=32)] member_count: Annotated[int, Field(ge=0, le=1_000_000)] parent_community_id: CommunityId | None = None @@ -591,7 +608,7 @@ def validate_span(self) -> HighlightSpan: class TextSearchHit(WireModel): record: RecordSummary bounded_excerpt: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8000), ] highlights: list[HighlightSpan] = Field(default_factory=list, max_length=100) @@ -600,15 +617,15 @@ class TextSearchHit(WireModel): class RuleCheckData(WireModel): matched_rules: list[RuleView] = Field(default_factory=list, max_length=50) - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) class ContextCompressData(WireModel): text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] original_tokens: Annotated[int, Field(ge=0, le=1_000_000)] @@ -625,7 +642,7 @@ class MemoryStoreBatchData(WireModel): class DocumentSource(WireModel): url: HttpsUrl - topic: MediumText + topic: UserMediumText content_hash: ContentHash @@ -637,13 +654,13 @@ class DocumentIngestData(WireModel): class VerifiedClaim(WireModel): - claim: MediumText + claim: UserMediumText status: Literal["supported", "contradicted", "unknown"] evidence_refs: list[EvidenceRef] = Field(default_factory=list, max_length=32) class Contradiction(WireModel): - claim: MediumText + claim: UserMediumText explanation: MediumText evidence_refs: list[EvidenceRef] = Field(min_length=1, max_length=32) @@ -657,11 +674,16 @@ class MemoryVerifyData(WireModel): class SandboxExecutionData(WireModel): success: bool - stdout: Annotated[str, StringConstraints(strict=True, max_length=100_000)] - stderr: Annotated[str, StringConstraints(strict=True, max_length=100_000)] + stdout: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] + stderr: Annotated[UserText, StringConstraints(strict=True, max_length=100_000)] exit_status: Annotated[int, Field(ge=-1, le=255)] execution_time_ms: Annotated[int, Field(ge=0, le=60_000)] - sanitized_logs: list[MediumText] = Field(default_factory=list, max_length=100) + # The same lines as stdout/stderr (see external_operations), so user text. + sanitized_logs: list[ + Annotated[ + UserText, StringConstraints(strict=True, min_length=1, max_length=4096) + ] + ] = Field(default_factory=list, max_length=100) class CodeIndexData(WireModel): @@ -675,7 +697,7 @@ class CodeIndexData(WireModel): class CodeEntitySummary(WireModel): code_entity_id: CodeEntityId kind: CodeEntityKind - qualified_name: NameText + qualified_name: UserNameText relative_file_path: RelativePath start_line: Annotated[int, Field(ge=1)] end_line: Annotated[int, Field(ge=1)] @@ -703,7 +725,7 @@ class TodoFinding(WireModel): relative_file_path: RelativePath line: Annotated[int, Field(ge=1)] todo_type: TodoType - text: MediumText + text: UserMediumText class CodeTodosStoreData(WireModel): @@ -714,7 +736,7 @@ class CodeTodosStoreData(WireModel): class RefactorProposalData(WireModel): proposal: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] affected_entities: list[CodeEntitySummary] = Field( @@ -729,7 +751,7 @@ class RelationshipView(WireModel): source_record_id: RecordId target_record_id: RecordId relationship_type: RelationshipType - description: OptionalMediumText = None + description: OptionalUserMediumText = None confidence: Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)] @@ -752,7 +774,7 @@ class MemoryChainTraceData(WireModel): class KnowledgeGraphNode(WireModel): record: RecordSummary - label: NameText + label: UserNameText node_type: Literal["record", "entity", "community"] @@ -769,7 +791,7 @@ class KnowledgeGraphData(WireModel): class KnowledgeGraphRenderData(WireModel): format: Literal["mermaid"] = "mermaid" text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] evidence_refs: list[EvidenceRef] = Field(default_factory=list, max_length=200) @@ -795,7 +817,7 @@ class CommunityRebuildData(WireModel): class EntitySummary(WireModel): entity_id: EntityId - name: NameText + name: UserNameText entity_type: EntityType mention_count: Annotated[int, Field(ge=0)] manifest_generation: Annotated[int, Field(ge=1)] @@ -810,7 +832,7 @@ class EntityBackfillData(WireModel): class EntityEvolutionItem(WireModel): happened_at: UtcDateTime - summary: MediumText + summary: UserMediumText event_id: EventId @@ -854,7 +876,15 @@ class ExportEvent(WireModel): event_type: NameText happened_at: UtcDateTime content_hash: ContentHash - payload: JsonObject + # Only the envelope's ``data`` holds user text; the rest stays guarded. + payload: UserJsonObject + + @model_validator(mode="after") + def guard_envelope(self) -> ExportEvent: + envelope = {k: v for k, v in self.payload.items() if k != "data"} + if contains_absolute_filesystem_path(envelope): + raise ValueError("absolute filesystem paths are forbidden on the v7 wire") + return self TransferSessionId = Annotated[ @@ -898,7 +928,7 @@ class ExportBundle(WireModel): page_proof: list[ContentHash] = Field(default_factory=list, max_length=32) next_cursor: Cursor | None = None complete: bool = True - legacy_rows: list[JsonObject] = Field(default_factory=list, max_length=4096) + legacy_rows: list[UserJsonObject] = Field(default_factory=list, max_length=4096) vector_points: list[PortableVectorPoint] = Field( default_factory=list, max_length=4096 ) @@ -970,7 +1000,7 @@ class WorkspaceLinkView(WireModel): workspace_id: WorkspaceId linked_workspace_id: WorkspaceId relationship: WorkspaceRelationship - label: OptionalMediumText = None + label: OptionalUserMediumText = None class WorkspaceConsolidateData(WireModel): @@ -1016,14 +1046,14 @@ class RuleEvolutionData(WireModel): class DebateRound(WireModel): round_number: Annotated[int, Field(ge=1, le=20)] - advocate: MediumText - challenger: MediumText + advocate: UserMediumText + challenger: UserMediumText class DecisionDebateData(WireModel): rounds: list[DebateRound] = Field(min_length=1, max_length=20) synthesis: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] consensus_record_id: RecordId @@ -1033,7 +1063,7 @@ class DecisionDebateData(WireModel): class SessionBriefInput(WireModel): workspace_id: WorkspaceId - focus_areas: list[ShortText] = Field(default_factory=list, max_length=10) + focus_areas: list[UserShortText] = Field(default_factory=list, max_length=10) warning_limit: Annotated[int, Field(ge=0, le=50)] = 10 failure_limit: Annotated[int, Field(ge=0, le=50)] = 10 @@ -1041,10 +1071,10 @@ class SessionBriefInput(WireModel): class MemoryPreflightInput(WireModel): workspace_id: WorkspaceId target_tool: ProtectedToolName - target_arguments: JsonObject + target_arguments: UserJsonObject description: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=1000), ] | None @@ -1054,7 +1084,7 @@ class MemoryPreflightInput(WireModel): class MemoryRecallInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] limit: Annotated[int, Field(ge=1, le=50)] = 10 @@ -1080,19 +1110,19 @@ def validate_candidate_limit(self) -> MemoryRecallInput: class MemoryStoreInput(WireModel): workspace_id: WorkspaceId record_type: RecordType - content: LongText + content: UserLongText rationale: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] | None ) = None - context: ContextJsonObject = Field(default_factory=dict) - tags: list[Tag] = Field(default_factory=list, max_length=32) + context: RecordContext = Field(default_factory=dict) + tags: UniqueTags = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None - procedure_steps: list[MediumText] = Field(default_factory=list, max_length=100) + procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1106,7 +1136,7 @@ def validate_procedure_steps(self) -> MemoryStoreInput: class EditPreflightInput(WireModel): workspace_id: WorkspaceId edit_request_id: EditRequestId - description: MediumText + description: UserMediumText class MemoryCaptureListInput(WireModel): @@ -1120,12 +1150,12 @@ class MemoryCapturePromoteInput(WireModel): candidate_id: CaptureCandidateId record_type: RecordType content: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=8_000), ] - rationale: OptionalMediumText = None - context: ContextJsonObject = Field(default_factory=dict) - tags: list[Tag] = Field(default_factory=list, max_length=16) + rationale: OptionalUserMediumText = None + context: RecordContext = Field(default_factory=dict) + tags: UniqueTags = Field(default_factory=list, max_length=16) idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1134,7 +1164,7 @@ class MemoryRecordOutcomeInput(WireModel): workspace_id: WorkspaceId record_id: RecordId outcome_text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] worked: bool @@ -1156,8 +1186,8 @@ class ActiveContextListInput(WireModel): class ContextTriggersMatchInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath | None = None - tags: list[Tag] = Field(default_factory=list, max_length=32) - entities: list[NameText] = Field(default_factory=list, max_length=32) + tags: UniqueTags = Field(default_factory=list, max_length=32) + entities: list[UserNameText] = Field(default_factory=list, max_length=32) limit: Annotated[int, Field(ge=1, le=20)] = 5 @@ -1182,7 +1212,7 @@ class MemoryRecallFileInput(WireModel): class MemoryRecallEntityInput(WireModel): workspace_id: WorkspaceId entity_id: EntityId | None = None - entity_name: NameText | None = None + entity_name: UserNameText | None = None entity_type: EntityType | None = None cursor: Cursor | None = None limit: Annotated[int, Field(ge=1, le=100)] = 20 @@ -1197,7 +1227,7 @@ def validate_selector(self) -> MemoryRecallEntityInput: class MemoryRecallHierarchicalInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] include_members: bool = False @@ -1207,7 +1237,7 @@ class MemoryRecallHierarchicalInput(WireModel): class MemorySearchTextInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] cursor: Cursor | None = None @@ -1218,36 +1248,36 @@ class MemorySearchTextInput(WireModel): class RuleCheckInput(WireModel): workspace_id: WorkspaceId - proposed_action: MediumText + proposed_action: UserMediumText context: ContextJsonObject = Field(default_factory=dict) class ContextCompressInput(WireModel): workspace_id: WorkspaceId text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=500_000), ] rate: Annotated[float, Field(ge=0.1, le=1, allow_inf_nan=False)] | None = None - content_type: ShortText | None = None + content_type: UserShortText | None = None preserve_code: bool = True class MemoryCreate(WireModel): record_type: RecordType - content: LongText + content: UserLongText rationale: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] | None ) = None - context: ContextJsonObject = Field(default_factory=dict) - tags: list[Tag] = Field(default_factory=list, max_length=32) + context: RecordContext = Field(default_factory=dict) + tags: UniqueTags = Field(default_factory=list, max_length=32) relative_file_path: RelativePath | None = None happened_at: AwareDateTime | None = None - procedure_steps: list[MediumText] = Field(default_factory=list, max_length=100) + procedure_steps: list[UserMediumText] = Field(default_factory=list, max_length=100) @model_validator(mode="after") def validate_procedure_steps(self) -> MemoryCreate: @@ -1268,7 +1298,7 @@ class MemoryLinkInput(WireModel): source_record_id: RecordId target_record_id: RecordId relationship_type: RelationshipType - description: OptionalMediumText = None + description: OptionalUserMediumText = None confidence: Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)] = 1.0 idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1296,7 +1326,7 @@ class MemoryPinSetInput(WireModel): class ActiveContextAddInput(WireModel): workspace_id: WorkspaceId record_id: RecordId - reason: OptionalMediumText = None + reason: OptionalUserMediumText = None priority: Annotated[int, Field(ge=-100, le=100)] = 0 expires_at: AwareDateTime | None = None preflight_token: PreflightToken @@ -1317,7 +1347,7 @@ class ActiveContextClearInput(WireModel): class DocumentIngestUrlInput(WireModel): workspace_id: WorkspaceId url: HttpsUrl - topic: MediumText + topic: UserMediumText chunk_size: Annotated[int, Field(ge=256, le=16_000)] = 2000 idempotency_key: IdempotencyKey preflight_token: PreflightToken @@ -1326,7 +1356,7 @@ class DocumentIngestUrlInput(WireModel): class MemoryVerifyInput(WireModel): workspace_id: WorkspaceId text: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] categories: RecordTypeSet | None = None @@ -1337,7 +1367,7 @@ class MemoryVerifyInput(WireModel): class SandboxExecutePythonInput(WireModel): workspace_id: WorkspaceId code: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=100_000), ] timeout_seconds: Annotated[int, Field(ge=1, le=60)] = 30 @@ -1356,7 +1386,7 @@ class CodeIndexInput(WireModel): class CodeSearchInput(WireModel): workspace_id: WorkspaceId query: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] cursor: Cursor | None = None @@ -1367,7 +1397,7 @@ class CodeSearchInput(WireModel): class CodeImpactAnalyzeInput(WireModel): workspace_id: WorkspaceId code_entity_id: CodeEntityId | None = None - qualified_name: NameText | None = None + qualified_name: UserNameText | None = None max_depth: Annotated[int, Field(ge=1, le=10)] = 3 @model_validator(mode="after") @@ -1401,27 +1431,27 @@ class CodeTodosScanAndStoreInput(WireModel): class CodeRefactorProposeInput(WireModel): workspace_id: WorkspaceId relative_file_path: RelativePath - objective: OptionalMediumText = None + objective: OptionalUserMediumText = None class RuleCreateInput(WireModel): workspace_id: WorkspaceId - trigger: MediumText - must_do: list[MediumText] = Field(default_factory=list, max_length=50) - must_not: list[MediumText] = Field(default_factory=list, max_length=50) - ask_first: list[MediumText] = Field(default_factory=list, max_length=50) - warnings: list[MediumText] = Field(default_factory=list, max_length=50) + trigger: UserMediumText + must_do: list[UserMediumText] = Field(default_factory=list, max_length=50) + must_not: list[UserMediumText] = Field(default_factory=list, max_length=50) + ask_first: list[UserMediumText] = Field(default_factory=list, max_length=50) + warnings: list[UserMediumText] = Field(default_factory=list, max_length=50) priority: Annotated[int, Field(ge=-1000, le=1000)] = 0 idempotency_key: IdempotencyKey preflight_token: PreflightToken class RulePatch(WireModel): - trigger: MediumText | None = None - must_do: list[MediumText] | None = Field(default=None, max_length=50) - must_not: list[MediumText] | None = Field(default=None, max_length=50) - ask_first: list[MediumText] | None = Field(default=None, max_length=50) - warnings: list[MediumText] | None = Field(default=None, max_length=50) + trigger: UserMediumText | None = None + must_do: list[UserMediumText] | None = Field(default=None, max_length=50) + must_not: list[UserMediumText] | None = Field(default=None, max_length=50) + ask_first: list[UserMediumText] | None = Field(default=None, max_length=50) + warnings: list[UserMediumText] | None = Field(default=None, max_length=50) priority: Annotated[int, Field(ge=-1000, le=1000)] | None = None enabled: bool | None = None @@ -1449,8 +1479,8 @@ class RuleListInput(WireModel): class ContextTriggerCreateInput(WireModel): workspace_id: WorkspaceId trigger_type: TriggerType - pattern: MediumText - recall_query: MediumText + pattern: UserMediumText + recall_query: UserMediumText categories: RecordTypeSet | None = None enabled: bool = True idempotency_key: IdempotencyKey @@ -1490,7 +1520,7 @@ class KnowledgeGraphGetInput(WireModel): record_ids: RecordIdSet500 | None = None query: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None @@ -1548,7 +1578,7 @@ class EntityBackfillInput(WireModel): class EntityEvolutionTraceInput(WireModel): workspace_id: WorkspaceId entity_id: EntityId | None = None - entity_name: NameText | None = None + entity_name: UserNameText | None = None entity_type: EntityType | None = None include_invalidated: bool = False @@ -1606,13 +1636,13 @@ class MemoryDuplicatesCleanupInput(MemoryDuplicatesPreviewInput): class MemoryCompactionPreviewInput(WireModel): workspace_id: WorkspaceId summary: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=50_000), ] limit: Annotated[int, Field(ge=1, le=100)] = 10 query: ( Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=2000), ] | None @@ -1674,7 +1704,7 @@ class WorkspaceLinkInput(WireModel): workspace_id: WorkspaceId linked_workspace_id: WorkspaceId relationship: WorkspaceRelationship = "related" - label: OptionalMediumText = None + label: OptionalUserMediumText = None preflight_token: PreflightToken @model_validator(mode="after") @@ -1740,13 +1770,13 @@ class RuleEvolutionAnalyzeInput(WireModel): class DecisionDebateInput(WireModel): workspace_id: WorkspaceId - topic: MediumText + topic: UserMediumText advocate_position: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] challenger_position: Annotated[ - str, + UserText, StringConstraints(strict=True, min_length=1, max_length=20_000), ] max_rounds: Annotated[int, Field(ge=1, le=20)] = 5 diff --git a/daem0nmcp/api/v7/utility_operations.py b/daem0nmcp/api/v7/utility_operations.py index db5deb2..2d07335 100644 --- a/daem0nmcp/api/v7/utility_operations.py +++ b/daem0nmcp/api/v7/utility_operations.py @@ -22,7 +22,7 @@ from ...workspace import Workspace, WorkspaceRegistry from .application import AdmittedRequest from .errors import STABLE_ERROR_CODE_SET -from .models import Page, contains_absolute_filesystem_path +from .models import Page from .runtime_protocols import WorkerPool from .tasks import await_task_terminal from .tools import ContextCompressData, RefactorProposalData, TodoFinding @@ -278,7 +278,7 @@ def _findings( if todo_type not in selected_types: continue rendered = line.strip()[:2000] - if not rendered or contains_absolute_filesystem_path(rendered): + if not rendered: continue try: findings.append( diff --git a/daem0nmcp/capabilities.py b/daem0nmcp/capabilities.py index 6b17e0e..71e8a5e 100644 --- a/daem0nmcp/capabilities.py +++ b/daem0nmcp/capabilities.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from importlib import metadata -from typing import Any +from typing import Any, Literal CAPABILITY_STATUSES = frozenset({"ready", "disabled", "degraded", "failed"}) CORE_DISTRIBUTIONS = ( @@ -117,10 +117,15 @@ def get(self, name: str) -> dict[str, Any]: configured = self._configured(profile) if configured is None: return self._failed(profile) - if not configured: + if configured is False: return self._disabled(profile) + # An unset variable means "on when installed": a profile that cannot run + # is simply off, while an explicit request for it is degraded. if name == "models-local" and sys.version_info < (3, 11): - return self._python_version_unavailable(profile) + capability = self._python_version_unavailable(profile) + if configured == "auto": + capability["status"] = "disabled" + return capability missing = [ distribution @@ -128,6 +133,8 @@ def get(self, name: str) -> dict[str, Any]: if not self._distribution_available(distribution) ] if missing: + if configured == "auto": + return self._not_installed(profile, missing) return self._degraded(profile, missing) return self._ready(profile) @@ -167,11 +174,13 @@ def _core(self) -> dict[str, Any]: }, } - def _configured(self, profile: CapabilityProfile) -> bool | None: - value = self._environ.get(profile.environment_key, "false").strip().lower() + def _configured(self, profile: CapabilityProfile) -> bool | Literal["auto"] | None: + value = self._environ.get(profile.environment_key, "").strip().lower() + if not value: + return "auto" if value in {"1", "true", "yes", "on"}: return True - if value in {"0", "false", "no", "off", ""}: + if value in {"0", "false", "no", "off"}: return False return None @@ -191,6 +200,24 @@ def _disabled(profile: CapabilityProfile) -> dict[str, Any]: "remediation": { "action": "enable_configuration", "environment": f"{profile.environment_key}=true", + "message": ( + f"Unset {profile.environment_key} or set it to true; the " + f"profile turns on when 'daem0nmcp[{profile.name}]' is installed." + ), + }, + } + + @staticmethod + def _not_installed( + profile: CapabilityProfile, missing: list[str] + ) -> dict[str, Any]: + return { + "name": profile.name, + "status": "disabled", + "remediation": { + "action": "install_extra", + "command": f"pip install 'daem0nmcp[{profile.name}]'", + "missing": missing, }, } @@ -225,7 +252,10 @@ def _failed(self, profile: CapabilityProfile) -> dict[str, Any]: "remediation": { "action": "fix_configuration", "environment": profile.environment_key, - "message": "Use a boolean value: true or false.", + "message": ( + "Use true or false, or leave it unset to enable the " + "profile automatically when its extra is installed." + ), }, } diff --git a/daem0nmcp/capture_candidates.py b/daem0nmcp/capture_candidates.py index 6856e17..9c56d43 100644 --- a/daem0nmcp/capture_candidates.py +++ b/daem0nmcp/capture_candidates.py @@ -151,8 +151,6 @@ def _validated_record(value: Mapping[str, Any]) -> dict[str, Any]: raise ValueError("capture tags are invalid") if len(tags) != len(set(tags)): raise ValueError("capture tags must be unique") - if contains_absolute_filesystem_path(record): - raise ValueError("capture record contains a public absolute path") return { "record_type": record["record_type"], "content": content, diff --git a/daem0nmcp/cli.py b/daem0nmcp/cli.py index 08aae92..e6cd2cb 100644 --- a/daem0nmcp/cli.py +++ b/daem0nmcp/cli.py @@ -352,6 +352,14 @@ def main(): metavar="MIGRATION_ID", help="Roll back the active migration run (default: latest)", ) + migrate_v7_parser.add_argument( + "--discard-v7-writes", + action="store_true", + help=( + "Roll back even though events recorded after activation would " + "become unreachable" + ), + ) migrate_v7_parser.add_argument( "--batch-size", type=_v7_batch_size, @@ -378,6 +386,19 @@ def main(): "--workspace-id", required=True, help="Registered opaque workspace ID" ) + compact_projections_parser = subparsers.add_parser( + "compact-projections", + help="Offline: drop every superseded projection generation and VACUUM", + ) + compact_projections_parser.add_argument( + "--workspace-id", required=True, help="Registered opaque workspace ID" + ) + compact_projections_parser.add_argument( + "--no-vacuum", + action="store_true", + help="Drop the superseded generations without reclaiming the file", + ) + rebuild_projection_parser = subparsers.add_parser( "rebuild-projection", help="Build and activate one v7 retrieval projection" ) @@ -647,7 +668,11 @@ def main(): service = MigrationV7Service(registry) try: if args.rollback is not None: - result = service.rollback(args.project_path, args.rollback) + result = service.rollback( + args.project_path, + args.rollback, + discard_v7_writes=args.discard_v7_writes, + ) elif args.apply: result = service.apply(args.project_path, batch_size=args.batch_size) else: @@ -689,7 +714,15 @@ def main(): "checkpoints": {}, "validation": {}, "warnings": [], - "error": {"code": exc.code, "message": exc.code}, + # Migration details stay redacted to the code, except the + # rollback refusal, whose detail is the operator signal: a + # count and the storage-relative candidate, never a host path. + "error": { + "code": exc.code, + "message": str(exc) + if exc.code == "ROLLBACK_WOULD_HIDE_WRITES" + else exc.code, + }, } exit_code = 2 if exc.code == "UNSAFE_MIGRATION_PATH" else 1 except Exception: @@ -723,8 +756,10 @@ def main(): ) if payload.get("active_generation") is not None: print(f"Generation: {payload['active_generation']}") + for warning in payload.get("warnings") or (): + print(f"Warning: {warning}") if payload.get("error"): - print(f"Error: {payload['error']['code']}") + print(f"Error: {payload['error']['message']}") sys.exit(exit_code) if args.command == "recover-consolidation": @@ -780,11 +815,16 @@ def main(): print(f"Runs: {len(payload['runs'])}") sys.exit(exit_code) - if args.command in {"projection-status", "rebuild-projection"}: + if args.command in { + "compact-projections", + "projection-status", + "rebuild-projection", + }: import sqlite3 from .retrieval.operations import ( ProjectionOperationError, + compact_projections, projection_status, rebuild_projection, ) @@ -812,7 +852,12 @@ def main(): update={"project_root": str(workspace.root)} ) storage_path = Path(workspace_settings.get_storage_path()) - with DatabaseFileLock(storage_path, "shared"): + # Compaction drops generations a running server could be reading, + # so it takes the offline exclusive lock. + lock_mode = ( + "exclusive" if args.command == "compact-projections" else "shared" + ) + with DatabaseFileLock(storage_path, lock_mode): active = resolve_active_database(storage_path) if active.format_version != 7: raise ProjectionOperationError("FORMAT_7_REQUIRED") @@ -823,6 +868,12 @@ def main(): connection.execute("PRAGMA foreign_keys=ON") if args.command == "projection-status": payload = projection_status(connection, workspace.workspace_id) + elif args.command == "compact-projections": + payload = compact_projections( + connection, + workspace.workspace_id, + vacuum=not args.no_vacuum, + ) else: builders = create_projection_builders( connection, @@ -1230,15 +1281,18 @@ async def run_watcher(): if args.json: print( json.dumps( - {"error": "tree-sitter-languages not installed", "indexed": 0} + { + "error": "tree-sitter-language-pack not installed", + "indexed": 0, + } ) ) else: print( - "ERROR: Code indexing requires tree-sitter-languages", + "ERROR: Code indexing requires tree-sitter-language-pack", file=sys.stderr, ) - print("Install with: pip install tree-sitter-languages") + print("Install with: pip install 'daem0nmcp[apps]'") sys.exit(1) project_path = Path(args.project_path or os.getcwd()).resolve() diff --git a/daem0nmcp/database.py b/daem0nmcp/database.py index 93c2a07..e325b34 100644 --- a/daem0nmcp/database.py +++ b/daem0nmcp/database.py @@ -209,6 +209,23 @@ def _run_migrations( self._migrated = True + def _v6_migration_ceiling(self, previous_schema_version: int) -> int | None: + """The migration ceiling for a retained format-6 store, else ``None``. + + A brand-new database whose creation crashed part-way through its own + migration run is pointerless and therefore reads as format 6, but it + holds no user rows and is finished by the recovery branch in + ``init_db``; capping it would leave it permanently incomplete. + """ + + from .migrations.schema import _LAST_V6_SCHEMA_VERSION + + if self.format_version != 6: + return None + if previous_schema_version >= 16 and not self._has_user_rows(): + return None + return _LAST_V6_SCHEMA_VERSION + async def init_db(self): """Initialize the database tables and run migrations.""" # Skip if already initialized @@ -224,7 +241,13 @@ async def init_db(self): # Run migrations first for existing databases (sync operation) # This happens BEFORE we create the async engine to avoid lock conflicts if not is_new_db: - self._run_migrations() + # A format-6 store is what `migrate-v7` snapshots and can roll back + # to. Applying the v7 ledger to it in place would rewrite the + # source without a backup, and its governance backfill then goes + # stale against later v6 edits, so cap it at the last v6 version. + self._run_migrations( + maximum_version=self._v6_migration_ceiling(previous_schema_version) + ) # Then create any new tables async with self.engine.begin() as conn: @@ -425,7 +448,14 @@ def _validate_database(self, *, format_version: int) -> None: foreign = list(connection.execute("PRAGMA foreign_key_check")) if integrity != ["ok"] or foreign: raise RuntimeError("DATABASE_INTEGRITY_FAILED") - if self._schema_version() < CURRENT_SCHEMA_VERSION: + from .migrations.schema import _LAST_V6_SCHEMA_VERSION + + # A retained format-6 store stops at the last v6 version on + # purpose: `migrate-v7` owns everything above it. + if format_version == 7: + if self._schema_version() < CURRENT_SCHEMA_VERSION: + raise RuntimeError("SCHEMA_MIGRATION_INCOMPLETE") + elif self._schema_version() < _LAST_V6_SCHEMA_VERSION: raise RuntimeError("SCHEMA_MIGRATION_INCOMPLETE") if format_version == 7: required = { diff --git a/daem0nmcp/edit_bridge.py b/daem0nmcp/edit_bridge.py index 4c02fdd..79f1445 100644 --- a/daem0nmcp/edit_bridge.py +++ b/daem0nmcp/edit_bridge.py @@ -558,11 +558,9 @@ def _issue_receipt_sync( description: str, ) -> EditReceipt: now_us = _datetime_us(self.clock()) - if ( - not isinstance(description, str) - or not 1 <= len(description) <= 2_000 - or contains_absolute_filesystem_path(description) - ): + # The description is user text (it may mention paths) and only its + # hash is stored. + if not isinstance(description, str) or not 1 <= len(description) <= 2_000: raise EditBridgeError("INVALID_ARGUMENT", "edit description rejected") if os.path.normcase(str(workspace.root.resolve())) != scope.canonical_workspace: raise EditBridgeError("TOKEN_SCOPE_MISMATCH", "workspace scope changed") diff --git a/daem0nmcp/event_store.py b/daem0nmcp/event_store.py index bfea017..e7328b6 100644 --- a/daem0nmcp/event_store.py +++ b/daem0nmcp/event_store.py @@ -11,6 +11,7 @@ import json import math import re +import sqlite3 import unicodedata from collections.abc import Iterator, Mapping from dataclasses import dataclass @@ -536,7 +537,10 @@ def _append(self, command: EventCommand, payload: dict[str, Any]) -> AppendedEve """, values, ) - except Exception as exc: + except sqlite3.IntegrityError as exc: + # Only a uniqueness/constraint collision is a stream conflict; I/O + # and lock errors propagate so callers do not advise a retry of a + # conflict that never happened. raise EventStreamConflict( "event identity or stream version is occupied" ) from exc @@ -1933,7 +1937,8 @@ def _append( "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", values, ) - except Exception as exc: + except sqlite3.IntegrityError as exc: + # As for memory events: only a constraint collision is a conflict. raise EventStreamConflict( "governance event identity or stream version is occupied" ) from exc diff --git a/daem0nmcp/migrations/schema.py b/daem0nmcp/migrations/schema.py index 8371fb5..115efd7 100644 --- a/daem0nmcp/migrations/schema.py +++ b/daem0nmcp/migrations/schema.py @@ -12,6 +12,7 @@ import time from datetime import datetime, timezone from pathlib import Path +from typing import Any from daem0nmcp.schema_version import CURRENT_SCHEMA_VERSION @@ -2821,6 +2822,162 @@ def _retained_path_workspace_id(value: object) -> str | None: return f"ws_{hashlib.sha256(root_key.encode('utf-8')).hexdigest()[:24]}" +def _retained_source_id(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + return value + + +def _retained_rule_states( + connection: sqlite3.Connection, + workspace_id: str, +) -> dict[str, tuple[int, dict[str, Any]]]: + """The canonical rule state each retained v6 rule row stands for.""" + + if not _table_exists(connection, "rules"): + return {} + states: dict[str, tuple[int, dict[str, Any]]] = {} + for row in connection.execute( + "SELECT id,trigger,must_do,must_not,ask_first,warnings,priority," + "enabled,created_at FROM rules ORDER BY id" + ).fetchall(): + source_id = _retained_source_id(row[0]) + if not isinstance(row[1], str) or not 1 <= len(row[1]) <= 2_000: + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + if ( + isinstance(row[6], bool) + or not isinstance(row[6], int) + or not -1_000 <= row[6] <= 1_000 + or row[7] not in (0, 1) + ): + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + public_id = _retained_public_mapping( + connection, workspace_id, "rule", source_id + ) + created_at_us = _retained_timestamp_us(row[8]) + states[public_id] = ( + source_id, + { + "rule_id": public_id, + "trigger": row[1], + "must_do": _retained_text_list(row[2]), + "must_not": _retained_text_list(row[3]), + "ask_first": _retained_text_list(row[4]), + "warnings": _retained_text_list(row[5]), + "priority": row[6], + "enabled": bool(row[7]), + "created_at_us": created_at_us, + "updated_at_us": created_at_us, + }, + ) + return states + + +def _retained_trigger_states( + connection: sqlite3.Connection, + workspace_id: str, +) -> tuple[dict[str, tuple[int, dict[str, Any]]], int]: + """The canonical trigger states, plus the count left to another project. + + v6 recorded the project root each trigger belongs to. When the database + records exactly one, it is this workspace's own history even if the project + directory has since been moved, so its triggers migrate. With several + roots, only the ones naming this workspace do. + """ + + if not _table_exists(connection, "context_triggers"): + return {}, 0 + rows = connection.execute( + "SELECT id,project_path,trigger_type,pattern,recall_topic,recall_categories," + "is_active,priority,created_at FROM context_triggers ORDER BY id" + ).fetchall() + from daem0nmcp.migrations.v7 import sole_recorded_project_root + + sole_root = sole_recorded_project_root(connection) + public_types = { + "file_pattern": "file", + "tag_match": "tag", + "entity_match": "entity", + } + states: dict[str, tuple[int, dict[str, Any]]] = {} + foreign = 0 + for row in rows: + source_id = _retained_source_id(row[0]) + # A trigger that records no project root names no workspace, so it + # stays foreign exactly as it was before the sole-root rule existed. + local = ( + isinstance(row[1], str) + and bool(row[1]) + and ( + row[1] == sole_root + or _retained_path_workspace_id(row[1]) == workspace_id + ) + ) + if not local: + foreign += 1 + continue + trigger_type = public_types.get(row[2]) + if trigger_type is None: + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + if ( + not isinstance(row[3], str) + or not 1 <= len(row[3]) <= 2_000 + or not isinstance(row[4], str) + or not 1 <= len(row[4]) <= 2_000 + or row[6] not in (0, 1) + or isinstance(row[7], bool) + or not isinstance(row[7], int) + ): + raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") + public_id = _retained_public_mapping( + connection, workspace_id, "trigger", source_id + ) + created_at_us = _retained_timestamp_us(row[8]) + states[public_id] = ( + source_id, + { + "trigger_id": public_id, + "trigger_type": trigger_type, + "pattern": row[3], + "recall_query": row[4], + "categories": _retained_text_list(row[5]), + "enabled": bool(row[6]), + "priority": row[7], + "created_at_us": created_at_us, + "updated_at_us": created_at_us, + "deleted_at_us": None, + }, + ) + return states, foreign + + +def _migration_authored_streams( + connection: sqlite3.Connection, workspace_id: str +) -> set[str]: + """The governance streams the retained backfill created, not v7 clients.""" + + return { + str(row[0]) + for row in connection.execute( + "SELECT stream_id FROM governance_events WHERE workspace_id=? " + "AND stream_version=1 AND actor_type='migration'", + (workspace_id,), + ) + } + + +def _governance_stream_exists( + connection: sqlite3.Connection, workspace_id: str, stream_id: str +) -> bool: + return ( + connection.execute( + "SELECT 1 FROM governance_events WHERE workspace_id=? AND stream_id=?", + (workspace_id, stream_id), + ).fetchone() + is not None + ) + + def backfill_retained_governance( connection: sqlite3.Connection, workspace_id: str, @@ -2840,159 +2997,202 @@ def backfill_retained_governance( raise RuntimeError("GOVERNANCE_SCHEMA_INCOMPLETE") store = GovernanceEventStore(connection, assume_transaction=True) inserted = 0 - if _table_exists(connection, "rules"): - rows = connection.execute( - "SELECT id,trigger,must_do,must_not,ask_first,warnings,priority," - "enabled,created_at FROM rules ORDER BY id" - ).fetchall() - for row in rows: - source_id = row[0] - if ( - isinstance(source_id, bool) - or not isinstance(source_id, int) - or source_id < 1 - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - public_id = _retained_public_mapping( - connection, workspace_id, "rule", source_id - ) + trigger_states, _foreign = _retained_trigger_states(connection, workspace_id) + for kind, projection, column, event_type, states in ( + ( + "rule", + "governance_rules", + "rule_id", + "rule.created", + _retained_rule_states(connection, workspace_id), + ), + ( + "trigger", + "governance_context_triggers", + "trigger_id", + "context_trigger.created", + trigger_states, + ), + ): + for public_id, (source_id, state) in states.items(): if ( connection.execute( - "SELECT 1 FROM governance_rules WHERE workspace_id=? AND rule_id=?", + f"SELECT 1 FROM {projection} WHERE workspace_id=? AND {column}=?", (workspace_id, public_id), ).fetchone() is not None ): continue - if ( - connection.execute( - "SELECT 1 FROM governance_events WHERE workspace_id=? " - "AND stream_id=?", - (workspace_id, public_id), - ).fetchone() - is not None - ): + if _governance_stream_exists(connection, workspace_id, public_id): raise RuntimeError("GOVERNANCE_BACKFILL_INTEGRITY_ERROR") - created_at_us = _retained_timestamp_us(row[8]) - if not isinstance(row[1], str) or not 1 <= len(row[1]) <= 2_000: - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - if ( - isinstance(row[6], bool) - or not isinstance(row[6], int) - or not -1_000 <= row[6] <= 1_000 - or row[7] not in (0, 1) - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - state = { - "rule_id": public_id, - "trigger": row[1], - "must_do": _retained_text_list(row[2]), - "must_not": _retained_text_list(row[3]), - "ask_first": _retained_text_list(row[4]), - "warnings": _retained_text_list(row[5]), - "priority": row[6], - "enabled": bool(row[7]), - "created_at_us": created_at_us, - "updated_at_us": created_at_us, - } store.append_and_project( GovernanceEventCommand( workspace_id=workspace_id, stream_id=public_id, - stream_kind="rule", - event_type="rule.created", - occurred_at_us=created_at_us, - recorded_at_us=created_at_us, + stream_kind=kind, + event_type=event_type, + occurred_at_us=state["created_at_us"], + recorded_at_us=state["created_at_us"], actor_type="migration", - correlation_id=f"migration21:rule:{source_id}", + correlation_id=f"migration21:{kind}:{source_id}", payload=state, expected_stream_version=1, ) ) inserted += 1 - if _table_exists(connection, "context_triggers"): - rows = connection.execute( - "SELECT id,project_path,trigger_type,pattern,recall_topic,recall_categories," - "is_active,priority,created_at FROM context_triggers ORDER BY id" - ).fetchall() - public_types = { - "file_pattern": "file", - "tag_match": "tag", - "entity_match": "entity", + return inserted + + +def reconcile_retained_governance( + connection: sqlite3.Connection, + workspace_id: str, + *, + now_us: int, +) -> dict[str, int]: + """Carry v6 rule and trigger edits made after an earlier backfill into v7. + + A legacy command can apply migration 21 to a v6 store, and the v6 tables + may then be edited again before `migrate-v7` runs. The backfill skips a + stream that already exists, so without this pass the edit or the deletion + would never reach v7 and a `must_not` could silently disappear. + + Only streams the backfill itself created are reconciled. A reactivated + candidate is the database that *was* live, so it also holds rules and + triggers authored through v7, which have no retained v6 row; inferring + "deleted in v6" from their absence would disable or delete them. The + creating event's `actor_type` separates the two: the backfill writes + `migration`, a v7 client writes `client`. + + The governance event vocabulary has `rule.updated` and + `context_trigger.deleted` but no `context_trigger.updated`, so a rule that + no longer exists in v6 is recorded as disabled, and a trigger whose + content changed in v6 is counted and reported instead of rewritten. + """ + + from daem0nmcp.event_store import ( + GovernanceEventCommand, + GovernanceEventStore, + ) + + if not _table_exists(connection, "governance_events"): + raise RuntimeError("GOVERNANCE_SCHEMA_INCOMPLETE") + store = GovernanceEventStore(connection, assume_transaction=True) + counts = { + "rules_updated": 0, + "rules_disabled": 0, + "triggers_deleted": 0, + "triggers_unreconciled": 0, + } + migrated = _migration_authored_streams(connection, workspace_id) + + rule_states = _retained_rule_states(connection, workspace_id) + for row in connection.execute( + "SELECT rule_id,trigger,must_do_json,must_not_json,ask_first_json," + "warnings_json,priority,enabled,stream_version,created_at_us " + "FROM governance_rules WHERE workspace_id=? ORDER BY rule_id", + (workspace_id,), + ).fetchall(): + public_id = str(row[0]) + if public_id not in migrated: + continue + entry = rule_states.get(public_id) + current = { + "rule_id": public_id, + "trigger": row[1], + "must_do": json.loads(row[2]), + "must_not": json.loads(row[3]), + "ask_first": json.loads(row[4]), + "warnings": json.loads(row[5]), + "priority": row[6], + "enabled": bool(row[7]), } - for row in rows: - source_id = row[0] - if ( - isinstance(source_id, bool) - or not isinstance(source_id, int) - or source_id < 1 - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - if _retained_path_workspace_id(row[1]) != workspace_id: - continue - public_id = _retained_public_mapping( - connection, workspace_id, "trigger", source_id + removed = entry is None + if removed: + desired = dict(current, enabled=False) + correlation = f"migration21:rule-removed:{public_id}" + else: + desired = { + key: value + for key, value in entry[1].items() + if key not in {"created_at_us", "updated_at_us"} + } + correlation = f"migration21:rule-updated:{entry[0]}" + if desired == current: + continue + store.append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id=public_id, + stream_kind="rule", + event_type="rule.updated", + occurred_at_us=now_us, + recorded_at_us=now_us, + actor_type="migration", + correlation_id=correlation, + payload={ + **desired, + "created_at_us": int(row[9]), + "updated_at_us": now_us, + }, + expected_stream_version=int(row[8]) + 1, ) + ) + counts["rules_disabled" if removed else "rules_updated"] += 1 + + trigger_states, _foreign = _retained_trigger_states(connection, workspace_id) + for row in connection.execute( + "SELECT trigger_id,trigger_type,pattern,recall_query,categories_json," + "enabled,priority,stream_version,created_at_us FROM " + "governance_context_triggers WHERE workspace_id=? AND deleted_at_us IS NULL " + "ORDER BY trigger_id", + (workspace_id,), + ).fetchall(): + public_id = str(row[0]) + if public_id not in migrated: + continue + entry = trigger_states.get(public_id) + if entry is not None: + # No `context_trigger.updated` event type exists, so a v6 edit (or + # a rowid reused by a different trigger) cannot be replayed. Count + # it so the operator is told rather than left with a stale pattern. + state = entry[1] if ( - connection.execute( - "SELECT 1 FROM governance_context_triggers " - "WHERE workspace_id=? AND trigger_id=?", - (workspace_id, public_id), - ).fetchone() - is not None - ): - continue - if ( - connection.execute( - "SELECT 1 FROM governance_events WHERE workspace_id=? " - "AND stream_id=?", - (workspace_id, public_id), - ).fetchone() - is not None - ): - raise RuntimeError("GOVERNANCE_BACKFILL_INTEGRITY_ERROR") - trigger_type = public_types.get(row[2]) - if trigger_type is None: - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - if ( - not isinstance(row[3], str) - or not 1 <= len(row[3]) <= 2_000 - or not isinstance(row[4], str) - or not 1 <= len(row[4]) <= 2_000 - or row[6] not in (0, 1) - or isinstance(row[7], bool) - or not isinstance(row[7], int) + state["trigger_type"] != row[1] + or state["pattern"] != row[2] + or state["recall_query"] != row[3] + or state["categories"] != json.loads(row[4]) + or state["enabled"] != bool(row[5]) + or state["priority"] != row[6] ): - raise RuntimeError("GOVERNANCE_BACKFILL_INVALID") - created_at_us = _retained_timestamp_us(row[8]) - state = { - "trigger_id": public_id, - "trigger_type": trigger_type, - "pattern": row[3], - "recall_query": row[4], - "categories": _retained_text_list(row[5]), - "enabled": bool(row[6]), - "priority": row[7], - "created_at_us": created_at_us, - "updated_at_us": created_at_us, - "deleted_at_us": None, - } - store.append_and_project( - GovernanceEventCommand( - workspace_id=workspace_id, - stream_id=public_id, - stream_kind="trigger", - event_type="context_trigger.created", - occurred_at_us=created_at_us, - recorded_at_us=created_at_us, - actor_type="migration", - correlation_id=f"migration21:trigger:{source_id}", - payload=state, - expected_stream_version=1, - ) + counts["triggers_unreconciled"] += 1 + continue + store.append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id=public_id, + stream_kind="trigger", + event_type="context_trigger.deleted", + occurred_at_us=now_us, + recorded_at_us=now_us, + actor_type="migration", + correlation_id=f"migration21:trigger-removed:{public_id}", + payload={ + "trigger_id": public_id, + "trigger_type": row[1], + "pattern": row[2], + "recall_query": row[3], + "categories": json.loads(row[4]), + "enabled": bool(row[5]), + "priority": row[6], + "created_at_us": int(row[8]), + "updated_at_us": now_us, + "deleted_at_us": now_us, + }, + expected_stream_version=int(row[7]) + 1, ) - inserted += 1 - return inserted + ) + counts["triggers_deleted"] += 1 + return counts def _has_retained_public_rows(connection: sqlite3.Connection) -> bool: diff --git a/daem0nmcp/migrations/v7.py b/daem0nmcp/migrations/v7.py index ac55d19..3835716 100644 --- a/daem0nmcp/migrations/v7.py +++ b/daem0nmcp/migrations/v7.py @@ -20,6 +20,7 @@ from collections.abc import Callable, Iterable from dataclasses import asdict, dataclass, field from datetime import datetime, timezone +from functools import partial from pathlib import Path from typing import Any from urllib.parse import quote @@ -43,10 +44,37 @@ resolve_active_database, write_active_pointer, ) -from ..workspace import WorkspaceRegistry, resolve_derived_path +from ..workspace import ( + WorkspaceRegistry, + is_workspace_relative_path, + normalize_resolved_path, + relative_to_root, + resolve_derived_path, +) +from .schema import ( + _retained_trigger_states, + reconcile_retained_governance, +) TARGET_FORMAT_VERSION = 7 DEFAULT_BATCH_SIZE = 500 +# Every table a v7 client can add user data to after activation. The +# three event ledgers are append-only (schema triggers enforce it), so a +# row count is an exact high-water mark; capture candidates are plain +# inserts holding unapproved user content, so a rise is reported too. +_POST_ACTIVATION_TABLES = ( + "memory_events", + "governance_events", + "workspace_link_events", + "memory_capture_candidates", +) +# v6 accepted empty content; the v7 bounded summary needs at least one +# character. ``api/v7/models.MIGRATED_EMPTY_CONTENT`` uses the same marker for +# rows migrated before this normalization existed. +_EMPTY_CONTENT = "" +# The v7 wire limits on a record's tags, which v6 did not have. +_MAX_TAGS = 32 +_MAX_TAG_CHARS = 80 _KNOWN_MEMORY_TYPES = { "decision", "pattern", @@ -146,7 +174,7 @@ def _quoted_identifier(name: str) -> str: def _readonly_connection(path: Path) -> sqlite3.Connection: - # Migration dry-run must not write any byte. On Windows the no-lock VFS + # A dry run never changes the source database. On Windows the no-lock VFS # keeps read-only connections off the ``-shm`` read marks. POSIX has no # equivalent: ``unix-none`` lacks shared memory, so it cannot open a WAL # database at all. There, with no ``-wal``, ``-shm`` or ``-journal`` the @@ -184,6 +212,13 @@ def _table_names(connection: sqlite3.Connection) -> list[str]: ] +def _has_column(connection: sqlite3.Connection, table: str, column: str) -> bool: + return any( + str(row[1]) == column + for row in connection.execute(f"PRAGMA table_info({_quoted_identifier(table)})") + ) + + def _table_exists(connection: sqlite3.Connection, name: str) -> bool: return ( connection.execute( @@ -610,12 +645,11 @@ def _partial_candidates(root: Path | None) -> list[Path]: def _sqlite_backup(source: Path, destination: Path) -> None: - source_uri = "file:" + quote(source.resolve().as_posix(), safe="/:") + "?mode=ro" - source_connection = sqlite3.connect(source_uri, uri=True) + # Opened through the same reader the dry run uses, so the two do not + # disagree about what "read-only" means beside the v6 file. + source_connection = _readonly_connection(source) target_connection = sqlite3.connect(destination) try: - source_connection.execute("PRAGMA foreign_keys=ON") - source_connection.execute("PRAGMA busy_timeout=0") source_connection.backup(target_connection) target_connection.commit() finally: @@ -639,27 +673,51 @@ def _integrity(path: Path) -> dict[str, Any]: return {"integrity_check": "ok", "foreign_key_violations": foreign} +# v6 columns that hold a path on the machine that wrote them. The lossless +# row rides inside every migrated event's payload, and an export bundle is a +# shareable artifact, so the value is replaced by a digest rather than copied +# to the wire. The original stays in `source.snapshot.db` and in the retained +# v6 tables, and the usable form is derived into `record.file_path_relative`. +_HOST_PATH_COLUMNS = frozenset({"file_path", "project_path", "database_path"}) +_DERIVED_PATH_COLUMNS = frozenset({"file_path_relative", "relative_file_path"}) + + +def _host_path_marker(value: str) -> dict[str, Any]: + return { + "$host_path": { + "length": len(value), + "sha256": hashlib.sha256(value.encode("utf-8")).hexdigest(), + "snapshot": "source.snapshot.db", + } + } + + +def _lossless_column(row: sqlite3.Row, table: str, name: str) -> Any: + value = row[name] + if table == "memories" and name == "vector_embedding" and isinstance(value, bytes): + return { + "$legacy_vector": { + "length": len(value), + "sha256": hashlib.sha256(value).hexdigest(), + "snapshot": "source.snapshot.db", + } + } + if isinstance(value, str) and value: + if name in _HOST_PATH_COLUMNS: + return _host_path_marker(value) + # v6 wrote `../outside/x.py` here for a file above the project, and + # the absolute path itself across Windows drives; neither may leave + # the host either. + if name in _DERIVED_PATH_COLUMNS and not is_workspace_relative_path(value): + return _host_path_marker(value) + return _encode_sqlite_value(value, table=table, column=name) + + def _lossless_row(row: sqlite3.Row, table: str) -> dict[str, Any]: return { "table": table, "columns": [ - [ - name, - ( - { - "$legacy_vector": { - "length": len(row[name]), - "sha256": hashlib.sha256(row[name]).hexdigest(), - "snapshot": "source.snapshot.db", - } - } - if table == "memories" - and name == "vector_embedding" - and isinstance(row[name], bytes) - else _encode_sqlite_value(row[name], table=table, column=name) - ), - ] - for name in tuple(row.keys()) + [name, _lossless_column(row, table, name)] for name in tuple(row.keys()) ], } @@ -715,8 +773,109 @@ def _legacy_score(value: Any) -> float | None: return number if math.isfinite(number) and 0 <= number <= 1 else None +def sole_recorded_project_root(connection: sqlite3.Connection) -> str | None: + """The one project root a v6 database records, or ``None``. + + v6 wrote the project root onto rows in several tables. When every one of + them names the same root, that root is this database's own project even + after the directory was moved; when they disagree, no single root can be + inferred. Spellings are compared normalized, so a trailing separator or + Windows case does not split one root into two. This is the single + definition: file links and context triggers both use it. + """ + + recorded: dict[str, str] = {} + for table in _table_names(connection): + if table in _V7_TABLE_NAMES: + continue + columns = { + str(row[1]) + for row in connection.execute( + f"PRAGMA table_info({_quoted_identifier(table)})" + ) + } + if "project_path" not in columns: + continue + for value in _distinct(connection, table, "project_path"): + if not isinstance(value, str) or not value: + continue + recorded.setdefault(_project_root_key(value), value) + if len(recorded) > 1: + return None + if len(recorded) != 1: + return None + return next(iter(recorded.values())) + + +def _project_root_key(value: str) -> str: + try: + resolved = normalize_resolved_path(Path(value).expanduser().resolve()) + except (OSError, RuntimeError, ValueError): + return os.path.normcase(value.rstrip("/\\")) + return os.path.normcase(str(resolved)) + + +def _legacy_project_roots( + connection: sqlite3.Connection, workspace_root: Path +) -> tuple[str, ...]: + """The roots a v6 file link may be relative to, workspace root first. + + When the database records exactly one project root and it is not this + workspace, the project was moved before the migration, so file links are + resolved against it as well. Containment is decided by + ``relative_to_root``, which compares ``os.path.normcase`` forms; on a + case-insensitive volume whose real spelling differs from the recorded one + the link is dropped and counted rather than guessed. + """ + + roots = [str(workspace_root)] + recorded_root = sole_recorded_project_root(connection) + if recorded_root is not None and _project_root_key( + recorded_root + ) != _project_root_key(roots[0]): + roots.append(recorded_root) + return tuple(roots) + + +def _legacy_relative_path(row: sqlite3.Row, roots: Iterable[str]) -> str | None: + """Resolve a v6 file link to a workspace-relative path, or drop it. + + v6 stored a host-absolute ``file_path`` plus a relative form that is + ``../outside/x.py`` for a file above the project root (and the absolute + path itself across Windows drives). Neither may be emitted, so the stored + relative form is kept when it already is workspace-relative and otherwise + re-derived from the absolute path. Both raw values stay in the event's + lossless ``legacy`` payload either way. + """ + + stored = _column(row, "file_path_relative") + if is_workspace_relative_path(stored): + return str(stored) + absolute = _column(row, "file_path") + if not isinstance(absolute, str) or not absolute: + return None + for root in roots: + relative = relative_to_root(absolute, root) + if relative is not None: + return relative + return None + + +def _legacy_tags(values: Iterable[Any]) -> list[str]: + """Clip v6 tags to the v7 wire limits; the raw list stays in ``legacy``.""" + + clipped: list[str] = [] + for value in values: + if not isinstance(value, str) or not value: + continue + tag = value[:_MAX_TAG_CHARS] + if tag not in clipped: + clipped.append(tag) + return clipped[:_MAX_TAGS] + + def _memory_state( - row: sqlite3.Row, *, base: dict[str, Any] | None = None + row: sqlite3.Row, *, roots: Iterable[str] = (), base: dict[str, Any] | None = None ) -> dict[str, Any]: category = _column(row, "category") if base is None else base["record_type"] legacy_type = None @@ -727,7 +886,7 @@ def _memory_state( else: legacy_type = base["legacy_type"] context, _ = _parse_legacy_json(_column(row, "context"), dict, {}) - tags, _ = _parse_legacy_json(_column(row, "tags"), list, []) + raw_tags, _ = _parse_legacy_json(_column(row, "tags"), list, []) worked_value = _column(row, "worked") worked = None if worked_value is None else _legacy_bool(worked_value) content = _column(row, "content", "") @@ -736,16 +895,16 @@ def _memory_state( return { "record_type": category, "legacy_type": legacy_type, - "content": content, + "content": content or _EMPTY_CONTENT, "rationale": _column(row, "rationale"), "context": context, - "tags": tags, - "file_path": base.get("file_path") - if base is not None - else _column(row, "file_path"), + "tags": _legacy_tags(raw_tags), + # The host-absolute v6 path never reaches the v7 record; it survives + # in the event's lossless ``legacy`` payload. + "file_path": None, "file_path_relative": base.get("file_path_relative") if base is not None - else _column(row, "file_path_relative"), + else _legacy_relative_path(row, roots), "keywords": base.get("keywords") if base is not None else _column(row, "keywords"), @@ -838,6 +997,7 @@ def _projected_memory_state(row: sqlite3.Row) -> dict[str, Any]: def _validate_retained_compatibility( connection: sqlite3.Connection, workspace_id: str, + roots: tuple[str, ...] = (), ) -> None: """Compare current retained-v6 rows to their unique live typed claims.""" @@ -875,7 +1035,7 @@ def _validate_retained_compatibility( (workspace_id, stream_id), ).fetchone() if projected is None or canonical_json_bytes( - _memory_state(source) + _memory_state(source, roots=roots) ) != canonical_json_bytes(_projected_memory_state(projected)): raise MigrationV7Error( "VALIDATION_FAILED", "current memory compatibility row diverges" @@ -1013,6 +1173,49 @@ def relationship_endpoint(value: Any) -> str: ) +def _migration_warnings(validation: dict[str, Any]) -> tuple[str, ...]: + """State every migrated fact that is not a faithful copy of the v6 row.""" + + warnings: list[str] = [] + dropped = int(validation.get("dropped_file_link_count", 0)) + if dropped: + warnings.append( + f"{dropped} memory file link(s) pointed outside the workspace and were " + "kept only in the legacy event payload" + ) + foreign = int(validation.get("foreign_trigger_count", 0)) + if foreign: + warnings.append( + f"{foreign} context trigger(s) belong to another project path and were " + "not migrated" + ) + updated = int(validation.get("rules_updated", 0)) + if updated: + warnings.append( + f"{updated} rule(s) changed in v6 after an earlier in-place upgrade and " + "were reconciled" + ) + disabled = int(validation.get("rules_disabled", 0)) + if disabled: + warnings.append( + f"{disabled} rule(s) removed in v6 after an earlier in-place upgrade " + "were disabled; no rule deletion event exists" + ) + deleted = int(validation.get("triggers_deleted", 0)) + if deleted: + warnings.append( + f"{deleted} context trigger(s) removed in v6 after an earlier in-place " + "upgrade were marked deleted" + ) + unreconciled = int(validation.get("triggers_unreconciled", 0)) + if unreconciled: + warnings.append( + f"{unreconciled} context trigger(s) were edited in v6 after an earlier " + "in-place upgrade and keep the earlier content; re-create them" + ) + return tuple(warnings) + + def _run_migration_16(path: Path, workspace_id: str) -> None: from .schema import run_migrations @@ -1220,6 +1423,22 @@ def _mark_candidate_active( raise MigrationV7Error( "ACTIVATION_STATE_INVALID", "candidate run is not ready" ) + validation_row = active.execute( + "SELECT validation_json FROM v7_migration_runs " + "WHERE migration_run_id=?", + (run_id,), + ).fetchone() + try: + validation = json.loads(validation_row[0] or "{}") + except (TypeError, ValueError): + validation = {} + if isinstance(validation, dict): + validation["activation_counts"] = self._post_activation_counts(active) + active.execute( + "UPDATE v7_migration_runs SET validation_json=? " + "WHERE migration_run_id=?", + (canonical_json_bytes(validation).decode("utf-8"), run_id), + ) active.execute( "UPDATE projection_manifests SET status='active', activated_at_us=? " "WHERE workspace_id=? AND status='ready' AND generation=1 " @@ -1290,7 +1509,7 @@ def _mark_candidate_rolled_back(self, candidate: Path, run_id: str) -> None: connection.close() def _recover_ready_pointer( - self, resolved, workspace_id: str + self, resolved, workspace_id: str, workspace_root: Path ) -> MigrationResult | None: """Finish activation when the pointer was durable before metadata commit.""" @@ -1313,6 +1532,7 @@ def _recover_ready_pointer( run_id, workspace_id, inventory, + workspace_root, retained_authority=prior_validation.get("authority") == "retained_v7", ) self._bootstrap_lexical(resolved.path, workspace_id) @@ -1414,7 +1634,7 @@ def apply( if upgrade is not None: return upgrade recovered = self._recover_ready_pointer( - resolved, workspace.workspace_id + resolved, workspace.workspace_id, workspace.root ) if recovered is not None: return recovered @@ -1451,12 +1671,14 @@ def apply( run_id, workspace.workspace_id, batch_size, + workspace.root, ) validation = self._finalize_candidate( candidate, run_id, workspace.workspace_id, inventory, + workspace.root, retained_authority=action == "reactivate", ) self._bootstrap_lexical(candidate, workspace.workspace_id) @@ -1521,6 +1743,7 @@ def apply( inventory=inventory, checkpoints=checkpoints, validation=validation, + warnings=_migration_warnings(validation), ) except MigrationInterrupted: raise @@ -1633,7 +1856,8 @@ def _initialize_candidate( snapshot_hash, int(inventory["max_schema_version"]), "source.snapshot.db", - "candidate.db.partial", + # The final published artifact, as verify-v7 expects. + "candidate.db", canonical_json_bytes(inventory).decode("utf-8"), now, now, @@ -1890,14 +2114,16 @@ def _import_candidate( run_id: str, workspace_id: str, batch_size: int, + workspace_root: Path, ) -> dict[str, Any]: connection = sqlite3.connect(candidate) connection.row_factory = sqlite3.Row try: connection.execute("PRAGMA foreign_keys=ON") connection.execute("PRAGMA synchronous=FULL") + roots = _legacy_project_roots(connection, workspace_root) for table, importer in ( - ("memories", self._import_memory_row), + ("memories", partial(self._import_memory_row, roots=roots)), ("facts", self._import_fact_row), ("memory_relationships", self._import_relationship_row), ): @@ -1976,12 +2202,14 @@ def _import_memory_row( row: sqlite3.Row, run_id: str, workspace_id: str, + *, + roots: tuple[str, ...], ) -> None: legacy_id = str(row["id"]) record_id = deterministic_id( "mem", "memory", workspace_id, "legacy", "memories", legacy_id ) - state = _memory_state(row) + state = _memory_state(row, roots=roots) stream_version = 0 versions = connection.execute( "SELECT * FROM memory_versions WHERE memory_id=? ORDER BY version_number, id", @@ -1996,7 +2224,7 @@ def _import_memory_row( } for version in versions: stream_version += 1 - version_state = _memory_state(version, base=state) + version_state = _memory_state(version, roots=roots, base=state) occurred, valid_quality, valid_original = _parse_legacy_time( _column(version, "valid_from") or _column(version, "changed_at") ) @@ -2335,6 +2563,7 @@ def _finalize_candidate( run_id: str, workspace_id: str, source_inventory: dict[str, Any], + workspace_root: Path, *, retained_authority: bool = False, ) -> dict[str, Any]: @@ -2342,12 +2571,25 @@ def _finalize_candidate( connection.row_factory = sqlite3.Row try: connection.execute("PRAGMA foreign_keys=ON") + roots = _legacy_project_roots(connection, workspace_root) connection.execute("BEGIN IMMEDIATE") now = self._clock_us() connection.execute( "UPDATE v7_migration_runs SET status='validating', updated_at_us=? WHERE migration_run_id=?", (now, run_id), ) + # A legacy command may already have backfilled governance into the + # v6 file; the v6 tables can have changed since. Carry those edits + # over before the candidate is declared ready. + # `_run_migration_16` already ran the ledger's own + # backfill on this candidate, so every retained rule and trigger + # has a public id and a stream before the comparison below. + governance = reconcile_retained_governance( + connection, workspace_id, now_us=now + ) + _foreign_trigger_states, foreign_triggers = _retained_trigger_states( + connection, workspace_id + ) quick = [ str(row[0]) for row in connection.execute("PRAGMA integrity_check") ] @@ -2471,7 +2713,7 @@ def _finalize_candidate( if table == "memories" and ( imported[2] != "legacy.memory_state_imported" or canonical_json_bytes(imported_payload.get("record")) - != canonical_json_bytes(_memory_state(source_row)) + != canonical_json_bytes(_memory_state(source_row, roots=roots)) ): raise MigrationV7Error( "VALIDATION_FAILED", @@ -2864,7 +3106,7 @@ def _finalize_candidate( ) if retained_authority: - _validate_retained_compatibility(connection, workspace_id) + _validate_retained_compatibility(connection, workspace_id, roots) expected_maps = { "memories": int(source_inventory["memory_count"]), @@ -2949,6 +3191,21 @@ def _finalize_candidate( now, ), ) + # Count every head row that recorded a file link of any shape and + # kept none. Versions inherit the head's link, so they cannot lose + # one independently. + dropped_file_links = 0 + if _table_exists(connection, "memories"): + for source_row in connection.execute("SELECT * FROM memories"): + recorded = any( + isinstance(value, str) and value + for value in ( + _column(source_row, "file_path"), + _column(source_row, "file_path_relative"), + ) + ) + if recorded and _legacy_relative_path(source_row, roots) is None: + dropped_file_links += 1 validation = { "integrity_check": "ok", "foreign_key_violations": [], @@ -2959,6 +3216,14 @@ def _finalize_candidate( "relationship_version_count": _count( connection, "memory_relationship_versions" ), + "rule_count": _count(connection, "governance_rules"), + "trigger_count": _count(connection, "governance_context_triggers"), + "rules_updated": governance["rules_updated"], + "rules_disabled": governance["rules_disabled"], + "triggers_deleted": governance["triggers_deleted"], + "triggers_unreconciled": governance["triggers_unreconciled"], + "foreign_trigger_count": foreign_triggers, + "dropped_file_link_count": dropped_file_links, } if retained_authority: validation["authority"] = "retained_v7" @@ -3005,10 +3270,78 @@ def _restore_pointer(self, storage: Path, previous) -> None: return write_active_pointer(storage, previous.pointer) + @staticmethod + def _post_activation_counts(connection: sqlite3.Connection) -> dict[str, int]: + """Row counts for every table a v7 client can add user data to.""" + + counts: dict[str, int] = {} + for table in _POST_ACTIVATION_TABLES: + if not _table_exists(connection, table): + continue + counts[table] = int( + connection.execute( + f"SELECT count(*) FROM {_quoted_identifier(table)}" + ).fetchone()[0] + ) + return counts + + def _events_after_activation(self, candidate: Path, run_id: str) -> int: + """Count user writes made to the candidate since activation. + + Rolling the pointer back to v6 makes them unreachable: the v6 file + never received them, and a later v6 write changes the run identity, so + the next `--apply` builds a different candidate. + + The comparison is against the row counts recorded when the candidate + was activated, not against a timestamp: a backward clock step (NTP, + a resumed VM) would otherwise hide exactly the writes this refuses to + discard. A store activated before this was recorded falls back to the + activation timestamp. + """ + + connection = sqlite3.connect(candidate) + try: + row = connection.execute( + "SELECT activated_at_us,validation_json FROM v7_migration_runs " + "WHERE migration_run_id=?", + (run_id,), + ).fetchone() + if row is None or row[0] is None: + return 0 + activated = int(row[0]) + try: + recorded = json.loads(row[1] or "{}").get("activation_counts") + except (TypeError, ValueError): + recorded = None + current = self._post_activation_counts(connection) + if isinstance(recorded, dict): + return sum( + max(0, count - int(recorded.get(table, 0))) + for table, count in current.items() + ) + total = 0 + for table in _POST_ACTIVATION_TABLES: + if table not in current or not _has_column( + connection, table, "recorded_at_us" + ): + continue + total += int( + connection.execute( + f"SELECT count(*) FROM {_quoted_identifier(table)} " + "WHERE recorded_at_us>?", + (activated,), + ).fetchone()[0] + ) + return total + finally: + connection.close() + def rollback( self, selector: str | os.PathLike[str] | None = None, migration_run_id: str | None = "latest", + *, + discard_v7_writes: bool = False, ) -> MigrationResult: workspace, storage = self._storage(selector) with DatabaseFileLock(storage, "exclusive"): @@ -3099,6 +3432,14 @@ def rollback( workspace.workspace_id, error_code="ROLLBACK_STATE_INVALID", ) + hidden = self._events_after_activation(resolved.path, run_id) + if hidden and not discard_v7_writes: + raise MigrationV7Error( + "ROLLBACK_WOULD_HIDE_WRITES", + f"{hidden} event(s) recorded since activation would become " + f"unreachable; they stay in {current_relative}. Re-run with " + "--discard-v7-writes to roll back anyway", + ) pointer = ActiveDatabasePointer( format_version=source_format, generation=resolved.generation + 1, @@ -3116,6 +3457,14 @@ def rollback( source_format=TARGET_FORMAT_VERSION, migration_run_id=run_id, active_generation=pointer.generation, + warnings=( + ( + f"{hidden} event(s) recorded since activation stay only in " + f"{current_relative}", + ) + if hidden + else () + ), ) diff --git a/daem0nmcp/opencode_assets/commands/commune.md b/daem0nmcp/opencode_assets/commands/commune.md new file mode 100644 index 0000000..27a2781 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/commune.md @@ -0,0 +1,13 @@ +--- +description: Start the Daem0n v7 session for a registered workspace +--- + +Call `daem0nmcp_session_brief(workspace_id="$ARGUMENTS")` immediately. The +argument must be the configured opaque `workspace_id`, not a path. + +After receiving the briefing results: +1. Report the session status, active warnings, and recent activity summary +2. Acknowledge that the v7 scoped session is active + +This slash-command filename is only a host shortcut. For v6 migration details, +use `docs/v6-to-v7-tools.json`. diff --git a/daem0nmcp/opencode_assets/commands/counsel.md b/daem0nmcp/opencode_assets/commands/counsel.md new file mode 100644 index 0000000..d8cd22f --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/counsel.md @@ -0,0 +1,24 @@ +--- +description: Request an exact Daem0n v7 preflight capability +--- + +Parse `$ARGUMENTS` as the intended protected v7 tool and its arguments, then +call: + +```text +daem0nmcp_memory_preflight( + workspace_id="", + target_tool="", + target_arguments={}, + description="$ARGUMENTS" +) +``` + +After receiving the preflight results: +1. Report any WARNINGS the user must know about +2. Report any FAILED APPROACHES that are relevant +3. Report any must_not constraints that apply +4. Preserve the token only for the exact request it authorizes + +Do not claim a generic edit authorization. The server binds the capability to +the workspace, principal, session, target tool, and canonical arguments. diff --git a/daem0nmcp/opencode_assets/commands/inscribe.md b/daem0nmcp/opencode_assets/commands/inscribe.md new file mode 100644 index 0000000..1359bb2 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/inscribe.md @@ -0,0 +1,31 @@ +--- +description: Store a replay-safe Daem0n v7 memory record +--- + +Store the following as a v7 memory record: + +$ARGUMENTS + +Choose `record_type` from the content: +- "decision" for architectural or design choices +- "pattern" for recurring approaches to follow +- "warning" for things to avoid +- "learning" for lessons from experience + +Create one stable idempotency key. Call `daem0nmcp_memory_preflight` with +`target_tool="memory_store"` and the exact target arguments, then call: + +```text +daem0nmcp_memory_store( + workspace_id="", + record_type="", + content="$ARGUMENTS", + rationale="", + idempotency_key="", + preflight_token="" +) +``` + +Report the returned `record_id`. After verification, use +`daem0nmcp_memory_record_outcome` with that ID and a separate stable +idempotency key. diff --git a/daem0nmcp/opencode_assets/commands/recall.md b/daem0nmcp/opencode_assets/commands/recall.md new file mode 100644 index 0000000..e372a59 --- /dev/null +++ b/daem0nmcp/opencode_assets/commands/recall.md @@ -0,0 +1,14 @@ +--- +description: Retrieve bounded memories from a Daem0n v7 workspace +--- + +Call `daem0nmcp_memory_recall(workspace_id="", query="$ARGUMENTS", limit=10)` immediately. + +Present the retrieved memories in a clear format: +- Show each record's content, type, and creation time +- Highlight any warnings or failed approaches +- Preserve evidence references and truncation indicators + +For service diagnostics use +`daem0nmcp_system_health(workspace_id="")`. Do not replace the +opaque workspace ID with a project path. diff --git a/daem0nmcp/opencode_install.py b/daem0nmcp/opencode_install.py index bddb83a..98f05b5 100644 --- a/daem0nmcp/opencode_install.py +++ b/daem0nmcp/opencode_install.py @@ -1,8 +1,8 @@ """Installer for OpenCode integration. -Creates .opencode/ directory structure, ensures opencode.json exists -at project root, and writes the TypeScript covenant enforcement plugin -for MCP server connectivity and hook discipline. +Creates the .opencode/ directory structure and slash-command files, ensures +opencode.json exists at project root, and writes the TypeScript covenant +enforcement plugin for MCP server connectivity and hook discipline. """ import argparse @@ -44,12 +44,14 @@ def select_opencode_interface(version: str, requested: str | None = None) -> str # Subdirectories to scaffold inside .opencode/ _OPENCODE_SUBDIRS = ["commands", "plugins", "agents"] -# The packaged TypeScript resource is the sole installer source of truth. -PLUGIN_TEMPLATE = ( - importlib.resources.files("daem0nmcp.opencode_assets") - .joinpath("daem0n.ts") - .read_text(encoding="utf-8") -) +# The packaged resources are the sole installer source of truth. +_ASSETS = importlib.resources.files("daem0nmcp.opencode_assets") +PLUGIN_TEMPLATE = _ASSETS.joinpath("daem0n.ts").read_text(encoding="utf-8") +COMMAND_TEMPLATES = { + command.name: command.read_text(encoding="utf-8") + for command in _ASSETS.joinpath("commands").iterdir() + if command.name.endswith(".md") +} def detect_clients(project_path: Path) -> dict[str, Any]: @@ -96,20 +98,45 @@ def _ensure_dir(path: Path, dry_run: bool) -> str: return "[create]" -def _ensure_file(path: Path, content: str, dry_run: bool, force: bool) -> str: +def _reject_links(path: Path, root: Path) -> None: + """Refuse a target that a link could move outside the project.""" + current = path + while True: + if current.is_symlink(): + raise OSError(f"refusing to write through the link {current}") + if current == root: + return + parent = current.parent + if parent == current: + raise OSError(f"{path} is outside {root}") + current = parent + + +def _ensure_file( + path: Path, content: str, dry_run: bool, force: bool, root: Path +) -> str: """Ensure a file exists with given content. Returns status string.""" + _reject_links(path, root) if path.exists(): if force: if not dry_run: - path.write_text(content, encoding="utf-8") + # Replace rather than write in place, so the new file is ours. + path.unlink() + _write_new(path, content) return "[overwrite]" return "[exists]" if not dry_run: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") + _reject_links(path.parent, root) + _write_new(path, content) return "[create]" +def _write_new(path: Path, content: str) -> None: + with path.open("x", encoding="utf-8") as handle: + handle.write(content) + + def _configured_template( installation: BridgeInstallation | None, ) -> dict[str, Any]: @@ -266,7 +293,7 @@ def install_opencode( lines.append("Configuration:") json_content = json.dumps(_configured_template(installation), indent=2) + "\n" json_path = root / "opencode.json" - status = _ensure_file(json_path, json_content, dry_run, force) + status = _ensure_file(json_path, json_content, dry_run, force, root) lines.append(f" {status} opencode.json") if ( installation is not None @@ -294,9 +321,16 @@ def install_opencode( # -- Ensure plugin file ------------------------------------------ plugin_path = opencode_root / "plugins" / "daem0n.ts" - status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force) + status = _ensure_file(plugin_path, PLUGIN_TEMPLATE, dry_run, force, root) lines.append(f" {status} .opencode/plugins/daem0n.ts") + # -- Ensure slash-command files ---------------------------------- + for name, content in sorted(COMMAND_TEMPLATES.items()): + status = _ensure_file( + opencode_root / "commands" / name, content, dry_run, force, root + ) + lines.append(f" {status} .opencode/commands/{name}") + # -- AGENTS.md status (do NOT create) ---------------------------- if oc["agents_md"]: lines.append(" [exists] AGENTS.md") diff --git a/daem0nmcp/retrieval/jobs.py b/daem0nmcp/retrieval/jobs.py index 4376bbd..f9f7a60 100644 --- a/daem0nmcp/retrieval/jobs.py +++ b/daem0nmcp/retrieval/jobs.py @@ -26,6 +26,26 @@ _HEARTBEAT_SHUTDOWN_GRACE_SECONDS = 1.5 +def is_lock_contention(exc: BaseException) -> bool: + """Return whether *exc* is SQLite lock contention a caller may retry.""" + + if not isinstance(exc, sqlite3.OperationalError): + return False + code = getattr(exc, "sqlite_errorcode", None) + if isinstance(code, int): + return code & 0xFF in { + sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_LOCKED, + } + # sqlite_errorcode is unavailable on supported Python 3.10 builds. + # Match only SQLite's fixed lock diagnostics, never arbitrary detail. + return str(exc).casefold() in { + "database is locked", + "database table is locked", + "database schema is locked", + } + + class ProjectionJobError(RuntimeError): """Owned, sanitized projection-job failure.""" @@ -250,7 +270,7 @@ def _heartbeat_loop( break except sqlite3.OperationalError as exc: connection.rollback() - if not self._is_lock_contention(exc): + if not is_lock_contention(exc): raise if stop.is_set(): if stop_seen_at is None: @@ -554,22 +574,6 @@ def _now(self) -> int: raise ProjectionJobError("PROJECTION_JOB_CLOCK_INVALID") return value - @staticmethod - def _is_lock_contention(exc: sqlite3.OperationalError) -> bool: - code = getattr(exc, "sqlite_errorcode", None) - if isinstance(code, int): - return code & 0xFF in { - sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_LOCKED, - } - # sqlite_errorcode is unavailable on supported Python 3.10 builds. - # Match only SQLite's fixed lock diagnostics, never arbitrary detail. - return str(exc).casefold() in { - "database is locked", - "database table is locked", - "database schema is locked", - } - def create_projection_job_runner( connection: sqlite3.Connection, @@ -593,4 +597,5 @@ def create_projection_job_runner( "ProjectionJobRun", "ProjectionJobRunner", "create_projection_job_runner", + "is_lock_contention", ] diff --git a/daem0nmcp/retrieval/legacy_compat.py b/daem0nmcp/retrieval/legacy_compat.py index dc405c6..4a80ac9 100644 --- a/daem0nmcp/retrieval/legacy_compat.py +++ b/daem0nmcp/retrieval/legacy_compat.py @@ -12,7 +12,6 @@ from pathlib import Path from typing import Any -from ..api.v7.models import contains_absolute_filesystem_path from ..event_store import ( event_hash_for, event_id_for_hash, @@ -57,9 +56,8 @@ def _safe_legacy_context(value: object) -> dict[str, Any] | None: ).encode("utf-8") except (TypeError, ValueError, RecursionError): return None - if len(encoded) > _MAX_LEGACY_CONTEXT_BYTES or contains_absolute_filesystem_path( - value - ): + # Context is user text and may mention paths (UD-3). + if len(encoded) > _MAX_LEGACY_CONTEXT_BYTES: return None # Detach the value from the parsed event payload before it reaches a cache. copied = json.loads(encoded.decode("utf-8")) diff --git a/daem0nmcp/retrieval/lexical_config.py b/daem0nmcp/retrieval/lexical_config.py index d23c243..98d5116 100644 --- a/daem0nmcp/retrieval/lexical_config.py +++ b/daem0nmcp/retrieval/lexical_config.py @@ -6,6 +6,13 @@ import json import re +# The projections a retrieval provider serves. ``projection_manifests`` also +# carries the migration's table snapshots (``memory_records`` and friends), so +# readers of retrieval freshness must select these names instead of refusing +# every other one. +RETRIEVAL_PROJECTION_NAMES = frozenset( + {"lexical", "dense", "graph", "temporal", "procedure", "outcome", "code"} +) LEXICAL_BM25_WEIGHTS = (1.0, 0.7, 1.5) LEXICAL_TOKENIZER = "unicode61 remove_diacritics 2" LEXICAL_BUILD_CONFIG = { @@ -18,6 +25,14 @@ "tokenizer": LEXICAL_TOKENIZER, } +# Per-generation row tables of the rebuildable local projections. Temporal +# keeps no rows of its own; graph generations are owned by discovery tables. +GENERATION_TABLES = { + "lexical": "retrieval_documents", + "procedure": "record_procedures", + "outcome": "record_outcome_view", +} + _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _FTS_PARTITION = re.compile(r"^retrieval_fts_[0-9a-f]{24}_g[1-9][0-9]*$") @@ -48,6 +63,8 @@ def lexical_fts_table_name(workspace_id: str, generation: int) -> str: __all__ = [ + "GENERATION_TABLES", + "RETRIEVAL_PROJECTION_NAMES", "LEXICAL_BM25_WEIGHTS", "LEXICAL_BUILD_CONFIG", "LEXICAL_TOKENIZER", diff --git a/daem0nmcp/retrieval/operations.py b/daem0nmcp/retrieval/operations.py index ab48297..08935e2 100644 --- a/daem0nmcp/retrieval/operations.py +++ b/daem0nmcp/retrieval/operations.py @@ -9,7 +9,12 @@ from dataclasses import asdict from typing import Any -from .projections import LexicalProjectionBuilder, ProjectionBuildError +from .projections import ( + COLLECTABLE_PROJECTIONS, + LexicalProjectionBuilder, + ProjectionBuildError, + collect_superseded_generations, +) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _PROJECTIONS = frozenset( @@ -157,8 +162,81 @@ def projection_status( } +def compact_projections( + connection: sqlite3.Connection, + workspace_id: str, + *, + vacuum: bool = True, +) -> dict[str, Any]: + """Drain every superseded generation offline, then reclaim the file. + + Online activations collect at most five generations each, so a database + that grew before the garbage collector existed needs about a hundred + writes to drain — and a ``VACUUM`` before that only rewrites the dead + generations. This runs the same collector in a loop until it reports + nothing left, so the following ``VACUUM`` actually reclaims the space. + The caller owns the offline exclusive lock; no activation may run + concurrently. + """ + + if not isinstance(connection, sqlite3.Connection): + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + _validate_workspace(workspace_id) + if connection.in_transaction: + raise ProjectionOperationError("PROJECTION_DATABASE_INVALID") + collected: dict[str, int] = {} + try: + before = _generation_counts(connection, workspace_id) + for projection in sorted(COLLECTABLE_PROJECTIONS): + removed = 0 + while True: + passed = collect_superseded_generations( + connection, workspace_id, projection + ) + if passed == 0: + break + removed += passed + collected[projection] = removed + if vacuum: + connection.execute("VACUUM") + after = _generation_counts(connection, workspace_id) + except sqlite3.Error as exc: + raise ProjectionOperationError("PROJECTION_COMPACTION_FAILED") from exc + return { + "collected": collected, + "superseded_manifests_after": after[0], + "superseded_manifests_before": before[0], + "vacuumed": bool(vacuum), + "workspace_id": workspace_id, + } + + +def _generation_counts( + connection: sqlite3.Connection, + workspace_id: str, +) -> tuple[int, int]: + placeholders = ",".join("?" for _ in COLLECTABLE_PROJECTIONS) + projections = sorted(COLLECTABLE_PROJECTIONS) + superseded = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders}) AND status<>'active'", + (workspace_id, *projections), + ).fetchone()[0] + ) + total = int( + connection.execute( + "SELECT COUNT(*) FROM projection_manifests WHERE workspace_id=? " + f"AND projection_name IN ({placeholders})", + (workspace_id, *projections), + ).fetchone()[0] + ) + return superseded, total + + __all__ = [ "ProjectionOperationError", + "compact_projections", "projection_status", "rebuild_projection", ] diff --git a/daem0nmcp/retrieval/projections.py b/daem0nmcp/retrieval/projections.py index 4114d29..2548719 100644 --- a/daem0nmcp/retrieval/projections.py +++ b/daem0nmcp/retrieval/projections.py @@ -4,6 +4,7 @@ import hashlib import json +import logging import re import sqlite3 import time @@ -11,14 +12,22 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import is_lock_contention from .lexical_config import ( + GENERATION_TABLES, LEXICAL_TOKENIZER, lexical_build_config_hash, lexical_fts_table_name, ) +from .specialized_contract import procedure_fts_table_name + +logger = logging.getLogger(__name__) _WORKSPACE_ID = re.compile(r"^ws_[0-9a-f]{24}$") _LEXICAL_BUILDER_VERSION = "retrieval-lexical-1" +# Graph is excluded: the discovery tables reference its generations. +COLLECTABLE_PROJECTIONS = frozenset({"lexical", "procedure", "outcome", "temporal"}) +_GC_BUSY_TIMEOUT_MS = 250 class ProjectionBuildError(RuntimeError): @@ -284,11 +293,17 @@ def rebuild( self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") if isinstance(exc, ProjectionBuildError): raise + if is_lock_contention(exc): + raise ProjectionBuildError( + "DATABASE_IN_USE", "lexical projection build was contended" + ) from exc if isinstance(exc, sqlite3.Error): raise ProjectionBuildError( "LEXICAL_UNAVAILABLE", "FTS5 projection build is unavailable" ) from exc raise + if owns_transaction: + collect_superseded_generations(self.connection, workspace_id, "lexical") return result def apply_active_delta(self, workspace_id: str, record_id: str) -> bool: @@ -748,8 +763,152 @@ def _clock_value(self) -> int: return value +_VICTIM_QUERY = """ +SELECT manifest_id, generation FROM projection_manifests +WHERE workspace_id=? AND projection_name=? AND typeof(generation)='integer' + AND ( + ( + status IN ('ready','failed') + AND manifest_id IS NOT ( + SELECT manifest_id FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='ready' + AND activated_at_us IS NOT NULL + ORDER BY generation DESC LIMIT 1 + ) + ) + OR ( + status='rebuild_required' + AND generation < COALESCE(( + SELECT generation FROM projection_manifests + WHERE workspace_id=? AND projection_name=? AND status='active' + AND typeof(generation)='integer' + ), -1) + ) + ) +ORDER BY generation LIMIT ? +""" + + +def _drop_generation( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + manifest_id: str, + generation: int, +) -> None: + fts_table = ( + lexical_fts_table_name(workspace_id, generation) + if projection == "lexical" + else procedure_fts_table_name(workspace_id, generation) + if projection == "procedure" + else None + ) + if fts_table is not None: + connection.execute(f'DROP TABLE IF EXISTS "{fts_table}"') + generation_table = GENERATION_TABLES.get(projection) + if generation_table is not None: + connection.execute( + f'DELETE FROM "{generation_table}" ' + "WHERE workspace_id=? AND projection_generation=?", + (workspace_id, generation), + ) + connection.execute( + "DELETE FROM projection_manifests WHERE manifest_id=?", + (manifest_id,), + ) + + +def collect_superseded_generations( + connection: sqlite3.Connection, + workspace_id: str, + projection: str, + limit: int = 5, +) -> int: + """Drop up to *limit* superseded generations of one local projection. + + Runs after an activation commits, in its own short transaction. Victims + are chosen by manifest status: ``ready`` and ``failed`` rows, plus + ``rebuild_required`` rows below the active generation (readers consult + those only while no active manifest exists). It never touches the active + or a ``building`` generation, nor the most recently deactivated one (the + highest activated ``ready`` generation), because a reader that resolved + the old manifest just before the switch may still be querying it. + + Best effort: one damaged generation is skipped rather than wedging the + rest, contention ends the pass, and nothing raises — the next activation + retries. Returns the number of generations removed. + """ + + if ( + projection not in COLLECTABLE_PROJECTIONS + or _WORKSPACE_ID.fullmatch(workspace_id) is None + or connection.in_transaction + ): + return 0 + collected = 0 + try: + previous_timeout = int(connection.execute("PRAGMA busy_timeout").fetchone()[0]) + connection.execute(f"PRAGMA busy_timeout={_GC_BUSY_TIMEOUT_MS}") + try: + connection.execute("BEGIN IMMEDIATE") + try: + victims = connection.execute( + _VICTIM_QUERY, + ( + workspace_id, + projection, + workspace_id, + projection, + workspace_id, + projection, + limit, + ), + ).fetchall() + for manifest_id, generation in victims: + connection.execute("SAVEPOINT retrieval_generation_gc") + try: + _drop_generation( + connection, + workspace_id, + projection, + str(manifest_id), + int(generation), + ) + except Exception as exc: + connection.execute( + "ROLLBACK TO SAVEPOINT retrieval_generation_gc" + ) + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + if is_lock_contention(exc): + raise + # A damaged generation must not wedge every later + # one; skip past it and keep collecting. + logger.warning( + "projection generation could not be collected", + exc_info=True, + ) + continue + connection.execute("RELEASE SAVEPOINT retrieval_generation_gc") + collected += 1 + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.execute(f"PRAGMA busy_timeout={previous_timeout}") + except Exception as exc: + if is_lock_contention(exc): + logger.debug("projection generation GC skipped: database busy") + else: + logger.warning("projection generation GC failed", exc_info=True) + return 0 + return collected + + __all__ = [ + "COLLECTABLE_PROJECTIONS", "LexicalProjectionBuilder", "ProjectionBuildError", "ProjectionBuildResult", + "collect_superseded_generations", ] diff --git a/daem0nmcp/retrieval/specialized_projection.py b/daem0nmcp/retrieval/specialized_projection.py index a5d8043..1282ee5 100644 --- a/daem0nmcp/retrieval/specialized_projection.py +++ b/daem0nmcp/retrieval/specialized_projection.py @@ -11,6 +11,8 @@ from dataclasses import dataclass from ..event_store import canonical_json_bytes, deterministic_id, sha256_json +from .jobs import is_lock_contention +from .projections import collect_superseded_generations from .specialized_contract import ( SPECIALIZED_BUILDER_VERSION, specialized_projection_contract, @@ -357,17 +359,28 @@ def rebuild( self.connection.commit() else: self.connection.execute(f"RELEASE SAVEPOINT {savepoint}") - return result except Exception as exc: self._rollback(owns_transaction, savepoint) if isinstance(exc, SpecializedProjectionBuildError): raise + if is_lock_contention(exc): + raise SpecializedProjectionBuildError( + "DATABASE_IN_USE", + "specialized projection build was contended", + ) from exc if isinstance(exc, sqlite3.Error): raise SpecializedProjectionBuildError( "PROJECTION_UNAVAILABLE", "specialized projection build is unavailable", ) from exc raise + if owns_transaction: + # Graph is not collectable: the discovery tables reference its + # generations. collect_superseded_generations enforces that. + collect_superseded_generations( + self.connection, workspace_id, projection_name + ) + return result def active_is_current(self, workspace_id: str, projection_name: str) -> bool: """Return whether the active specialized generation is exact.""" diff --git a/daem0nmcp/verification_v7.py b/daem0nmcp/verification_v7.py index be1003b..ea8c56a 100644 --- a/daem0nmcp/verification_v7.py +++ b/daem0nmcp/verification_v7.py @@ -35,6 +35,10 @@ ) from .migrations import MIGRATIONS from .migrations.v7 import _source_row_hash, inventory_database +from .retrieval.lexical_config import ( + GENERATION_TABLES, + RETRIEVAL_PROJECTION_NAMES, +) from .schema_version import CURRENT_SCHEMA_VERSION, REQUIRED_V7_SCHEMA_VERSIONS from .storage_activation import ( ActiveDatabasePointer, @@ -74,6 +78,19 @@ "workspace_link_events", } ) +# The snapshots `migrate-v7` activates for the tables it fills directly; no +# retrieval provider serves them, so nothing rebuilds them and the live +# manifest invariants cannot hold once anything is written. The migration's +# `entities`/`communities` manifests are written `rebuild_required`, so they +# keep their own row-count verification. +_TABLE_SNAPSHOT_PROJECTIONS = frozenset( + { + "memory_records", + "memory_fact_versions", + "memory_relationship_versions", + } +) +_DISCOVERY_PROJECTIONS = frozenset({"entities", "communities"}) _LOCAL_PROJECTIONS = frozenset( { "memory_records", @@ -86,17 +103,9 @@ "outcome", } ) -_SUPPORTED_PROJECTIONS = _LOCAL_PROJECTIONS | { - "dense", - "code", - "entities", - "communities", -} -_GENERATION_TABLES = { - "lexical": "retrieval_documents", - "procedure": "record_procedures", - "outcome": "record_outcome_view", -} +_SUPPORTED_PROJECTIONS = ( + _TABLE_SNAPSHOT_PROJECTIONS | _DISCOVERY_PROJECTIONS | RETRIEVAL_PROJECTION_NAMES +) class VerificationV7Error(RuntimeError): @@ -664,6 +673,34 @@ def _verify_mappings(connection: sqlite3.Connection, replay: sqlite3.Connection) return count +def _generation_inventory(connection: sqlite3.Connection) -> dict[str, int]: + """Count superseded local generations so a stalled GC is visible.""" + + from .retrieval.projections import COLLECTABLE_PROJECTIONS + + projections = sorted(COLLECTABLE_PROJECTIONS) + placeholders = ",".join("?" for _ in projections) + superseded = { + str(row[0]): int(row[1]) + for row in connection.execute( + "SELECT projection_name,COUNT(*) FROM projection_manifests " + f"WHERE projection_name IN ({placeholders}) AND status<>'active' " + "GROUP BY projection_name", + projections, + ) + } + inventory = {name: superseded.get(name, 0) for name in projections} + inventory["fts_partitions"] = int( + connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*' " + "AND (name GLOB 'retrieval_fts_*_g*' " + "OR name GLOB 'retrieval_procedure_fts_*_g*')" + ).fetchone()[0] + ) + return inventory + + def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: from .retrieval.projections import LexicalProjectionBuilder from .retrieval.specialized_projection import SpecializedProjectionBuilder @@ -679,7 +716,7 @@ def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: "source_event_root_hash,details_json,generation,row_count," "cursor_recorded_at_us,cursor_event_id FROM projection_manifests" ).fetchall() - for name, generation_table in _GENERATION_TABLES.items(): + for name, generation_table in GENERATION_TABLES.items(): if connection.execute( f'SELECT 1 FROM "{generation_table}" rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " @@ -709,6 +746,12 @@ def _verify_manifests(connection: sqlite3.Connection) -> tuple[int, int, int]: previous = latest_local.get(key) if previous is None or int(row[6]) > previous[0]: latest_local[key] = (int(row[6]), not current or declared_stale) + if projection_name in _TABLE_SNAPSHOT_PROJECTIONS: + # A migration snapshot records what `migrate-v7` built for a table + # the event store owns, not a rebuildable retrieval generation, so + # later writes move the event cursor and the row count past it by + # design. Only the retrieval projections carry those invariants. + continue if row[2] == "active" and not declared_stale: if projection_name == "code": if (row[8], row[9]) != (None, None): @@ -934,6 +977,14 @@ def _verify_database( ) except Exception as exc: checks["projection_manifests"] = _check(False, error=type(exc).__name__) + try: + # Superseded generations should stay near zero. A number that + # keeps growing means the activation-time GC is failing. + checks["projection_generations"] = _check( + True, **_generation_inventory(source) + ) + except Exception as exc: + checks["projection_generations"] = _check(False, error=type(exc).__name__) try: invalid = int( source.execute( @@ -1115,7 +1166,7 @@ def _refresh_manifests(candidate: Path, workspace_ids: list[str]) -> None: ) # Remove only orphaned local generations in the candidate. Otherwise a # renamed/deleted manifest can leave an FTS table occupying the next ID. - for name, table in _GENERATION_TABLES.items(): + for name, table in GENERATION_TABLES.items(): connection.execute( f'DELETE FROM "{table}" AS rows WHERE NOT EXISTS ' "(SELECT 1 FROM projection_manifests manifest WHERE " diff --git a/daem0nmcp/workspace.py b/daem0nmcp/workspace.py index f3ce449..e598f02 100644 --- a/daem0nmcp/workspace.py +++ b/daem0nmcp/workspace.py @@ -5,10 +5,14 @@ import hashlib import json import os +import posixpath +import re from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path +_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:") + class WorkspaceAccessError(ValueError): """Raised when a caller selects a workspace that is not registered.""" @@ -182,6 +186,55 @@ def resolve(self, selector: str | os.PathLike[str] | None) -> Workspace: return workspace +def is_workspace_relative_path(value: object) -> bool: + """Return whether a string is a normalized workspace-relative POSIX path.""" + + if not isinstance(value, str) or not 1 <= len(value) <= 1024: + return False + if value == ".": + return True + if ( + "\\" in value + or "\x00" in value + or value.startswith(("/", "~")) + or _WINDOWS_DRIVE.match(value) is not None + ): + return False + if any(component in {"", ".", ".."} for component in value.split("/")): + return False + return posixpath.normpath(value) == value + + +def relative_to_root( + path: str | os.PathLike[str], + root: str | os.PathLike[str], +) -> str | None: + """Return ``path`` inside ``root`` as a POSIX relative path, else ``None``. + + Containment is decided with ``Path.relative_to`` on resolved, normalized + paths, never with a string prefix, so ``C:\\proj2\\a.py`` is not treated as + living under ``C:\\proj``. The returned string always satisfies the wire's + relative-path rule. + """ + + try: + resolved_root = normalize_resolved_path(Path(root).expanduser().resolve()) + resolved = normalize_resolved_path(Path(path).expanduser().resolve()) + depth = len( + Path(os.path.normcase(str(resolved))) + .relative_to(os.path.normcase(str(resolved_root))) + .parts + ) + except (OSError, RuntimeError, ValueError): + return None + if depth < 1: + return None + # Join the resolved path's own tail so the result keeps the filesystem's + # spelling rather than the case-folded form containment was decided on. + value = "/".join(resolved.parts[-depth:]) + return value if is_workspace_relative_path(value) else None + + def _has_parent_component(value: str) -> bool: normalized = value.replace("\\", "/") return ".." in normalized.split("/") diff --git a/docs/release/v7/inventory.md b/docs/release/v7/inventory.md index db82039..cc683e0 100644 --- a/docs/release/v7/inventory.md +++ b/docs/release/v7/inventory.md @@ -2,7 +2,7 @@ Generated by `python scripts/v7_release_inventory.py --write`. Registration is not acceptance; only an `accepted` requirement with explicit evidence metadata can be accepted. -Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **571**. +Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resource templates: **10**. v6 mappings: **64**. Atomic requirements: **572**. ## Release ledger @@ -131,6 +131,7 @@ Production entry point: `daem0nmcp.server:main`. Registered tools: **75**. Resou | hook | `hooks/daem0n_stop_hook.py` | `hooks/daem0n_stop_hook.py` | Hook hooks/daem0n_stop_hook.py runs with its documented input and redacted failure behavior. — pending evidence | pending | | cli | `briefing` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli briefing parses and exercises its documented result contract. — pending evidence | pending | | cli | `check` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli check parses and exercises its documented result contract. — pending evidence | pending | +| cli | `compact-projections` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract. — pending evidence | pending | | cli | `index` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli index parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-claude-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-claude-hooks parses and exercises its documented result contract. — pending evidence | pending | | cli | `install-hooks` | `daem0nmcp.cli:main` | python -m daem0nmcp.cli install-hooks parses and exercises its documented result contract. — pending evidence | pending | diff --git a/docs/release/v7/requirements.json b/docs/release/v7/requirements.json index 6346d97..8b452d4 100644 --- a/docs/release/v7/requirements.json +++ b/docs/release/v7/requirements.json @@ -4370,6 +4370,16 @@ "package": "daem0nmcp.cli", "status": "pending" }, + { + "acceptance": "python -m daem0nmcp.cli compact-projections parses and exercises its documented result contract.", + "entry_point": "daem0nmcp.cli:main", + "evidence": [], + "id": "cli.compact-projections", + "kind": "cli", + "owner": "cli", + "package": "daem0nmcp.cli", + "status": "pending" + }, { "acceptance": "python -m daem0nmcp.cli index parses and exercises its documented result contract.", "entry_point": "daem0nmcp.cli:main", @@ -5744,8 +5754,8 @@ "tests/test_legacy_vector_migration_guards.py" ], "observed_surface": { - "atomic_requirements": 571, - "cli_commands": 20, + "atomic_requirements": 572, + "cli_commands": 21, "dependency_profiles": 10, "hooks": 15, "registered_tools": 75, diff --git a/docs/v7-operations.md b/docs/v7-operations.md index c7e8b8e..b344dfa 100644 --- a/docs/v7-operations.md +++ b/docs/v7-operations.md @@ -197,9 +197,39 @@ the dry-run inventory. `FUTURE_V7_SCHEMA`, `UNSUPPORTED_V7_SCHEMA`, and Daem0nMCP build; they are not repaired automatically. After an interrupted run, repeat the same `migrate-v7 --apply` command. Use `migrate-v7 --rollback latest` to return to the retained predecessor and `--apply` to reactivate its verified -candidate. The generic `migrate` command remains an in-place additive migration -entry point; use `migrate-v7 --apply` when retention, verification, atomic -activation, and rollback are required. +candidate. Rollback refuses with `ROLLBACK_WOULD_HIDE_WRITES`, naming the count +and the candidate that retains them, when events were recorded after +activation; `--discard-v7-writes` proceeds and reports what stayed behind. The +generic `migrate` command remains an in-place additive migration entry point, +capped at the last v6 schema version while a store is still format 6 so it +cannot rewrite what `migrate-v7` snapshots; use `migrate-v7 --apply` when +retention, verification, atomic activation, and rollback are required. + +Stop the server before applying or rolling back: both commands take the +storage lock, and a running server on the old pointer sees the generation +change. Until a v6 workspace is migrated, `session_brief` answers +`MIGRATION_REQUIRED` (non-retryable) with the command to run, and every other +tool is blocked behind it with `COMMUNION_REQUIRED`. The export/import path +answers `MIGRATION_REQUIRED` too, without the command, because the generic +router returns the code alone. The four data resources answer the single +invariant `RESOURCE_UNAVAILABLE` for every failure by design, so that a caller +cannot enumerate workspaces by reading their errors; `session_brief` is where +the reason is named. + +The migration keeps the v6 row exactly as it was inside the migrated +database — in the event log's `legacy` payload and in the retained v6 tables — +but that copy is not part of `workspace_export`, which carries the v7 +projection. Host paths are never exported: the columns that hold one are +recorded as a digest, and the usable form is derived into +`relative_file_path`. A workspace migrated before this release keeps the raw +value inside its own hash-chained events, so its exports need a re-migration +to be path-free; reads, recall, outcomes and resources do not. + +`migrate-v7 --apply` reports what it could not copy faithfully in `warnings` +and counts it in `validation`: memory file links that pointed outside the +workspace (kept only in the event's legacy payload), context triggers +belonging to another project path, and rules or triggers reconciled because +the v6 tables changed after an earlier in-place upgrade. `verify-v7` checks authority and projections; `--repair-projections` performs offline replay and atomic activation. Rollback retains the v7 write history diff --git a/pyproject.toml b/pyproject.toml index 09a50ed..e906eb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ include = ["daem0nmcp*"] "static/*.js", "static/renderers/*.js", ] -"daem0nmcp.opencode_assets" = ["*.ts"] +"daem0nmcp.opencode_assets" = ["*.ts", "commands/*.md"] # Ruff configuration [tool.ruff] diff --git a/start_daem0nmcp_server.bat b/start_daem0nmcp_server.bat index fb71a2c..08214d1 100644 --- a/start_daem0nmcp_server.bat +++ b/start_daem0nmcp_server.bat @@ -3,13 +3,26 @@ REM ============================================ REM Daem0nMCP HTTP Server Launcher for Windows REM ============================================ REM This script starts the Daem0nMCP HTTP server -REM for use with Claude Code on Windows. +REM for one project on Windows. REM -REM Windows has known issues with stdio transport, -REM so HTTP transport is required. +REM Usage: start_daem0nmcp_server.bat [project_dir] +REM The project defaults to an exported +REM DAEM0NMCP_PROJECT_ROOT, else the directory you +REM run it from. Stdio (python -m daem0nmcp) also +REM works on Windows; use this launcher when you +REM want an HTTP server instead. REM ============================================ title Daem0nMCP Server +REM Delayed expansion would re-parse ! and ^ in a project path +setlocal DisableDelayedExpansion + +REM Capture the project before changing directory +set "DAEM0N_PROJECT=" +if not "%~1"=="" set "DAEM0N_PROJECT=%~f1" +if "%~1"=="" if not defined DAEM0NMCP_PROJECT_ROOT set "DAEM0N_PROJECT=%CD%" +if /i "%DAEM0N_PROJECT%"=="%SystemRoot%" goto :system_directory +if /i "%DAEM0N_PROJECT%"=="%SystemRoot%\System32" goto :system_directory REM Change to the script's directory cd /d "%~dp0" @@ -25,8 +38,13 @@ echo \^|/ echo * echo. -REM Start the server -python start_server.py --port 9876 +REM Start the server. An exported DAEM0NMCP_PROJECT_ROOT is left alone. +REM The trailing "\." keeps a drive root such as C:\ from escaping the quote +if defined DAEM0N_PROJECT ( + python start_server.py --port 9876 --project "%DAEM0N_PROJECT%\." +) else ( + python start_server.py --port 9876 +) REM If the server exits, pause so user can see any errors if errorlevel 1 ( @@ -34,3 +52,12 @@ if errorlevel 1 ( echo [ERROR] Server exited with an error. Press any key to close. pause >nul ) +goto :eof + +:system_directory +echo. +echo [ERROR] Refusing to serve "%DAEM0N_PROJECT%". +echo Run the launcher from your project, or pass it: +echo start_daem0nmcp_server.bat C:\path\to\project +pause >nul +exit /b 2 diff --git a/start_server.py b/start_server.py index ef3ffb7..dd5f5cf 100644 --- a/start_server.py +++ b/start_server.py @@ -22,12 +22,22 @@ def main(): parser.add_argument("--host", default="127.0.0.1", help="Host to bind to") parser.add_argument( "--project", - help="Project directory for storage (default: current directory)", + help=( + "Project directory to serve (default: DAEM0NMCP_PROJECT_ROOT if set, " + "else the current directory)" + ), ) args = parser.parse_args() - project_root = Path(args.project or os.getcwd()).resolve() - os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(project_root) + project_root = Path( + args.project or os.environ.get("DAEM0NMCP_PROJECT_ROOT") or Path.cwd() + ) + if not project_root.is_dir(): + # Otherwise the server silently creates and serves an empty workspace + # wherever a typo points. + parser.error(f"project directory does not exist: {project_root}") + os.environ["DAEM0NMCP_PROJECT_ROOT"] = str(project_root.resolve()) + print(f"Serving project: {project_root.resolve()}", file=sys.stderr) # Import only after the workspace environment is fixed. Both public # launchers use the same v7 composition and transport-security boundary. diff --git a/tests/api_v7/process_client.py b/tests/api_v7/process_client.py index 1f2f795..4dca20a 100644 --- a/tests/api_v7/process_client.py +++ b/tests/api_v7/process_client.py @@ -150,6 +150,29 @@ def server_environment( return environment +_LOG_TAIL_BYTES = 8000 + + +def dump_server_log(path: Path) -> None: + """Print the server's own log when a process test fails. + + CI shows only the assertion, so an INTERNAL_ERROR or a crash inside the + server is otherwise invisible; the tmp directory is gone by then. + """ + + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError as error: + print(f"\n----- {path.name} unavailable: {error} -----", flush=True) + return + tail = text[-_LOG_TAIL_BYTES:] + print( + f"\n----- {path.name} (last {len(tail)} of {len(text)} chars) -----\n{tail}" + f"\n----- end {path.name} -----", + flush=True, + ) + + @asynccontextmanager async def process_client( workspace: Path, @@ -160,6 +183,7 @@ async def process_client( environment_overrides: dict[str, str] | None = None, http_probe: Callable[[str], Awaitable[None]] | None = None, http_headers: dict[str, str] | None = None, + log_name: str = "stdio-server.log", ): """Launch the installed module in a workspace and clean up on every exit.""" environment = server_environment( @@ -176,7 +200,7 @@ async def process_client( ) body_error: BaseException | None = None try: - with (workspace / "stdio-server.log").open("w", encoding="utf-8") as log: + with (workspace / log_name).open("w", encoding="utf-8") as log: async with ( stdio_client(parameters, errlog=log) as (read, write), ClientSession( @@ -191,10 +215,12 @@ async def process_client( # SDK stdout-reader BrokenResourceError during shutdown. body_error = error except BaseException as cleanup_error: + dump_server_log(workspace / log_name) if body_error is not None: raise body_error from cleanup_error raise if body_error is not None: + dump_server_log(workspace / log_name) raise body_error return if transport != "streamable-http": @@ -202,68 +228,79 @@ async def process_client( with socket.socket() as reservation: reservation.bind(("127.0.0.1", 0)) port = reservation.getsockname()[1] - with (workspace / "http-server.log").open("w", encoding="utf-8") as log: - process = subprocess.Popen( - [ - python, - "-m", - "daem0nmcp.server", - "--transport", - transport, - "--host", - "127.0.0.1", - "--port", - str(port), - ], - cwd=workspace, - env=environment, - stdout=log, - stderr=log, - creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0, - ) - try: - deadline = asyncio.get_running_loop().time() + 30 - while True: - if process.poll() is not None: - raise RuntimeError( - f"MCP server exited with code {process.returncode}; inspect http-server.log" - ) - try: - _, writer = await asyncio.open_connection("127.0.0.1", port) - writer.close() - await writer.wait_closed() - break - except OSError: - if asyncio.get_running_loop().time() >= deadline: - raise TimeoutError( - "MCP HTTP server did not become ready" - ) from None - await asyncio.sleep(0.05) - if http_probe is not None: - await http_probe(f"http://127.0.0.1:{port}/mcp") - async with ( - httpx.AsyncClient(headers=http_headers, timeout=30) as http_client, - streamable_http_client( - f"http://127.0.0.1:{port}/mcp", http_client=http_client - ) as ( - read, - write, - _, - ), - ClientSession( - read, write, read_timeout_seconds=timedelta(seconds=30) - ) as session, - ): - await session.initialize() - yield session - finally: - if process.poll() is None: - process.terminate() + failed = False + try: + with (workspace / "http-server.log").open("w", encoding="utf-8") as log: + process = subprocess.Popen( + [ + python, + "-m", + "daem0nmcp.server", + "--transport", + transport, + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=workspace, + env=environment, + stdout=log, + stderr=log, + creationflags=subprocess.CREATE_NO_WINDOW + if sys.platform == "win32" + else 0, + ) try: - await asyncio.to_thread(process.wait, timeout=10) - except subprocess.TimeoutExpired: - process.kill() - await asyncio.to_thread(process.wait, timeout=10) + deadline = asyncio.get_running_loop().time() + 30 + while True: + if process.poll() is not None: + raise RuntimeError( + f"MCP server exited with code {process.returncode}; inspect http-server.log" + ) + try: + _, writer = await asyncio.open_connection("127.0.0.1", port) + writer.close() + await writer.wait_closed() + break + except OSError: + if asyncio.get_running_loop().time() >= deadline: + raise TimeoutError( + "MCP HTTP server did not become ready" + ) from None + await asyncio.sleep(0.05) + if http_probe is not None: + await http_probe(f"http://127.0.0.1:{port}/mcp") + async with ( + httpx.AsyncClient(headers=http_headers, timeout=30) as http_client, + streamable_http_client( + f"http://127.0.0.1:{port}/mcp", http_client=http_client + ) as ( + read, + write, + _, + ), + ClientSession( + read, write, read_timeout_seconds=timedelta(seconds=30) + ) as session, + ): + await session.initialize() + yield session + except BaseException: + failed = True + raise + finally: + if process.poll() is None: + process.terminate() + try: + await asyncio.to_thread(process.wait, timeout=10) + except subprocess.TimeoutExpired: + process.kill() + await asyncio.to_thread(process.wait, timeout=10) + finally: + if failed: + # Outside the log context, so the tail is flushed and complete. + dump_server_log(workspace / "http-server.log") async def call(session: ClientSession, tool: str, arguments: dict) -> dict: diff --git a/tests/api_v7/test_application.py b/tests/api_v7/test_application.py index 023a3d1..d20677d 100644 --- a/tests/api_v7/test_application.py +++ b/tests/api_v7/test_application.py @@ -129,6 +129,59 @@ async def operation(*, workspace: Workspace, request: object) -> object: self.assertEqual(second.error.code, "TOKEN_REPLAYED") self.assertEqual(len(calls), 1) + async def test_database_in_use_is_retryable_and_points_at_a_fresh_preflight( + self, + ) -> None: + from daem0nmcp.api.v7.models import MutationReceipt + + class BusyError(RuntimeError): + code = "DATABASE_IN_USE" + + attempts: list[object] = [] + + async def operation(*, workspace: Workspace, request: object) -> object: + del workspace + attempts.append(request) + if len(attempts) == 1: + raise BusyError("database is locked") + return MutationReceipt( + operation_id="op_archive_retry", + affected_ids=[request.record_id], + event_ids=[], + counts={"changed": 1}, + idempotent_replay=False, + ) + + self.gate.record_briefing(self.scope) + target = { + "workspace_id": WORKSPACE_ID, + "record_id": RECORD_ID, + "archived": True, + } + handler = self._router({"memory_archive_set": operation}).handler( + "memory_archive_set" + ) + token = self.gate.issue_preflight(self.scope, "memory_archive_set", target) + + busy = await handler(**target, preflight_token=token) + self.assertFalse(busy.ok) + self.assertEqual(busy.error.code, "DATABASE_IN_USE") + self.assertTrue(busy.error.retryable) + self.assertEqual(busy.error.retry_after_ms, 250) + self.assertEqual(busy.error.remedy.tool, "memory_preflight") + self.assertEqual( + busy.error.remedy.arguments["target_arguments"], + {"record_id": RECORD_ID, "archived": True}, + ) + # The token was spent before the operation ran: a literal retry is + # refused, and the remedy (a fresh preflight) succeeds. + replayed = await handler(**target, preflight_token=token) + self.assertEqual(replayed.error.code, "TOKEN_REPLAYED") + fresh = self.gate.issue_preflight(self.scope, "memory_archive_set", target) + retried = await handler(**target, preflight_token=fresh) + self.assertTrue(retried.ok) + self.assertEqual(len(attempts), 2) + async def test_protected_durable_execution_requires_exact_worker_context( self, ) -> None: diff --git a/tests/api_v7/test_discovery_operations.py b/tests/api_v7/test_discovery_operations.py index a40b2d8..452ccc8 100644 --- a/tests/api_v7/test_discovery_operations.py +++ b/tests/api_v7/test_discovery_operations.py @@ -1162,6 +1162,85 @@ async def retrieve(self, workspace, query, linked_workspace_ids): self.assertEqual(1, len(second.items)) self.assertNotEqual(first.items[0].record_id, second.items[0].record_id) + async def test_memory_recall_entity_retries_only_a_pending_catch_up( + self, + ) -> None: + """Only a projection that still owes the records is worth retrying.""" + from daem0nmcp.api.v7.discovery_operations import DiscoveryOperationError + from daem0nmcp.api.v7.models import ( + ProviderDiagnostic, + RetrievalData, + TokenUsage, + ) + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._activate_discovery() + + class RecallService: + """Returns nothing, as a lexical miss on the entity name does.""" + + async def retrieve(self, workspace, query, linked_workspace_ids): + return RetrievalData( + provider_diagnostics=[ + ProviderDiagnostic( + provider="lexical", + status="unavailable", + manifest_generation=None, + elapsed_ms=0.0, + reason="LEXICAL_UNAVAILABLE", + returned_count=0, + ) + ], + abstained=True, + abstention_reason="NO_EVIDENCE", + token_usage=TokenUsage( + budget=query.token_budget, + requested=0, + selected=0, + rendered=0, + dropped=0, + ), + ) + + operation = self._operations(recall_service=RecallService())[ + "memory_recall_entity" + ] + request = _request( + "memory_recall_entity", + workspace_id=self.workspace.workspace_id, + entity_name="Authentication", + ) + + async def code() -> str: + with self.assertRaises(DiscoveryOperationError) as raised: + await operation(workspace=self.workspace, request=request) + return raised.exception.code + + # The records are not in the lexical projection yet and its rebuild + # job is queued: a retry can make progress. + self.assertEqual("DATABASE_IN_USE", await code()) + + # The rebuild dead-lettered, so nothing will catch up on its own. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute( + "UPDATE background_jobs SET status='dead_letter'," + "finished_at_us=1 WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + + # The projection holds every member, so the miss is genuine even + # though lexical reports itself unavailable and a job is queued. + with closing(sqlite3.connect(self.database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + LexicalProjectionBuilder(connection).rebuild(self.workspace.workspace_id) + connection.execute( + "UPDATE background_jobs SET status='queued',finished_at_us=NULL " + "WHERE job_type='retrieval.projection_rebuild'" + ) + connection.commit() + self.assertEqual("CAPABILITY_DEGRADED", await code()) + async def test_stats_read_the_active_canonical_graph_snapshot(self) -> None: """Counting retained-v6 graph rows would misreport the v7 projection.""" from daem0nmcp.api.v7.tools import KnowledgeGraphStatsData @@ -1586,6 +1665,30 @@ def factory(): self.assertFalse(loaded) self.assertEqual("CAPABILITY_DISABLED", raised.exception.code) self.assertEqual("apps", raised.exception.capability_states[0].name) + self.assertIn( + "daem0nmcp[apps]", raised.exception.capability_states[0].remediation + ) + + async def test_each_apps_status_names_its_own_remedy(self) -> None: + """An invalid setting is a configuration fix, not a missing package.""" + from daem0nmcp.api.v7.discovery_operations import _code_capability_error + + remedies = {} + for status in ("disabled", "degraded", "failed"): + error = _code_capability_error( + SimpleNamespace(capability_statuses={"apps": status}) + ) + assert error is not None + remedies[status] = error.capability_states[0].remediation + + self.assertIn("unless DAEM0NMCP_APPS_ENABLED=false", remedies["disabled"]) + self.assertEqual(remedies["degraded"], remedies["disabled"]) + self.assertIn("true, false, or leave it unset", remedies["failed"]) + self.assertIsNone( + _code_capability_error( + SimpleNamespace(capability_statuses={"apps": "ready"}) + ) + ) async def test_code_index_source_change_rolls_back_before_publish(self) -> None: from daem0nmcp.api.v7.discovery_operations import ( diff --git a/tests/api_v7/test_entrypoints.py b/tests/api_v7/test_entrypoints.py index dcd0b15..fdad6c8 100644 --- a/tests/api_v7/test_entrypoints.py +++ b/tests/api_v7/test_entrypoints.py @@ -2,7 +2,9 @@ import io import os +import shutil import sys +import tempfile import types import unittest from contextlib import redirect_stdout @@ -15,6 +17,9 @@ def test_http_launcher_builds_remote_v7_server_and_uses_shared_runner(self) -> N import start_server from daem0nmcp.api.v7.launcher import ServerOptions + workspace = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, workspace, True) + built: list[tuple[str, str | None]] = [] launched: list[tuple[object, ServerOptions]] = [] server = object() @@ -26,7 +31,7 @@ def create_server( ) -> object: self.assertEqual( os.environ.get("DAEM0NMCP_PROJECT_ROOT"), - str(Path("tests/http-workspace").resolve()), + str(workspace.resolve()), ) built.append((transport, host)) return server @@ -40,7 +45,7 @@ def create_server( "--port", "9988", "--project", - str(Path("tests/http-workspace")), + str(workspace), ] output = io.StringIO() with ( diff --git a/tests/api_v7/test_error_honesty.py b/tests/api_v7/test_error_honesty.py new file mode 100644 index 0000000..d248f3b --- /dev/null +++ b/tests/api_v7/test_error_honesty.py @@ -0,0 +1,279 @@ +"""Transient contention is retryable and every internal error is logged (KD-6).""" + +from __future__ import annotations + +import asyncio +import json +import logging +import sqlite3 +import threading +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from fastmcp import Client + +from daem0nmcp.api.v7 import runtime_services +from daem0nmcp.api.v7.production import ( + _briefing_reader, + _guidance_reader, + create_v7_server, +) +from daem0nmcp.api.v7.resource_repository import build_sqlite_resource_readers +from daem0nmcp.api.v7.runtime_services import ( + BasicBriefingService, + BasicPreflightService, + RuntimeServiceError, +) +from daem0nmcp.api.v7.tools import SessionBriefInput +from daem0nmcp.bounded_workers import BoundedWorkerPool +from daem0nmcp.config import Settings +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import initialize_workspaces + +FLOWS = 8 + + +def _server(root: Path): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(root), + workspace_roots=[str(root)], + dream_enabled=False, + ), + environ={}, + ) + + +def _database(root: Path) -> Path: + return resolve_active_database(root / ".daem0nmcp" / "storage").path + + +def _is_retryable_busy(response: dict[str, Any]) -> bool: + error = response["error"] + return ( + error["code"] == "DATABASE_IN_USE" + and error["retryable"] is True + and error["retry_after_ms"] == 250 + ) + + +class _Session: + def __init__(self, client: Client, workspace_id: str) -> None: + self._client = client + self.workspace_id = workspace_id + + async def call(self, tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await self._client.call_tool( + tool, + {"workspace_id": self.workspace_id, **arguments}, + raise_on_error=False, + ) + assert isinstance(result.structured_content, dict), tool + return result.structured_content + + async def preflight_and_store(self, index: int) -> dict[str, Any]: + arguments = { + "record_type": "decision", + "content": f"Concurrent decision number {index}", + "idempotency_key": f"concurrent-store-{index:04d}", + } + preflight = await self.call( + "memory_preflight", + {"target_tool": "memory_store", "target_arguments": arguments}, + ) + if not preflight["ok"]: + return preflight + return await self.call( + "memory_store", + {**arguments, "preflight_token": preflight["data"]["preflight_token"]}, + ) + + +async def test_parallel_stores_are_ok_or_retryable_and_land_exactly_once(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + + async def flow(index: int) -> list[dict[str, Any]]: + observed = [] + deadline = asyncio.get_running_loop().time() + 60 + while True: + response = await session.preflight_and_store(index) + observed.append(response) + if response["ok"]: + return observed + assert _is_retryable_busy(response), response + assert asyncio.get_running_loop().time() < deadline, observed + await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) + + results = await asyncio.gather(*(flow(index) for index in range(FLOWS))) + + for observed in results: + assert observed[-1]["ok"], observed + # Logged, not asserted: a fast host may never contend. + print("DATABASE_IN_USE retries:", sum(len(observed) - 1 for observed in results)) + with closing(sqlite3.connect(_database(tmp_path))) as connection: + records = connection.execute("SELECT count(*) FROM memory_records").fetchone() + events = connection.execute( + "SELECT count(*) FROM memory_events WHERE event_type='memory.created'" + ).fetchone() + assert (records[0], events[0]) == (FLOWS, FLOWS) + + +async def test_store_behind_a_held_write_lock_is_retryable_database_in_use(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + with closing( + sqlite3.connect(_database(tmp_path), isolation_level=None) + ) as holder: + holder.execute("BEGIN IMMEDIATE") + try: + # Waits out the writer's 5 s SQLite busy timeout. + blocked = await session.preflight_and_store(1) + finally: + holder.execute("ROLLBACK") + assert not blocked["ok"] + assert _is_retryable_busy(blocked), blocked + retried = await session.preflight_and_store(1) + assert retried["ok"], retried + + +async def test_internal_error_is_logged_under_the_returned_correlation_id( + tmp_path, monkeypatch, caplog +): + def explode(*_arguments, **_keywords): + raise RuntimeError("induced-failure-canary") + + # Raised inside the writer transaction, so it reaches the log only through + # the MEMORY_STORE_FAILED exception chain. + monkeypatch.setattr(runtime_services, "_existing_idempotent_event", explode) + [workspace] = await initialize_workspaces((tmp_path,)) + async with Client(_server(tmp_path)) as client: + session = _Session(client, workspace.workspace_id) + assert (await session.call("session_brief", {}))["ok"] + with caplog.at_level(logging.ERROR, logger="daem0nmcp.api.v7.responses"): + response = await session.preflight_and_store(1) + + assert not response["ok"] + error = response["error"] + assert (error["code"], error["retryable"]) == ("INTERNAL_ERROR", False) + assert "induced-failure-canary" not in json.dumps(response) + [record] = [ + record + for record in caplog.records + if record.name == "daem0nmcp.api.v7.responses" + ] + assert error["correlation_id"] in record.getMessage() + assert "induced-failure-canary" in caplog.text + assert "MEMORY_STORE_FAILED" in caplog.text + + +@pytest.mark.parametrize("contention", ["full_pool", "overrun_read"]) +@pytest.mark.parametrize("service", ["memory_preflight", "session_brief"]) +async def test_contended_briefing_snapshot_is_retryable_database_in_use( + tmp_path, service, contention +): + """The CI failure path: guidance/briefing -> read_briefing_snapshot -> _run. + + MCP resource URIs still report an opaque access failure; the repository + code only matters on these two service paths. + """ + [workspace] = await initialize_workspaces((tmp_path,)) + storage = tmp_path / ".daem0nmcp" / "storage" + pool = BoundedWorkerPool(max_workers=1, thread_name_prefix="test-contention") + release = threading.Event() + + def resolve(_workspace): + if contention == "overrun_read": + release.wait(timeout=30) # overrun the 0.05 s deadline, then succeed + return resolve_active_database(storage) + + readers = build_sqlite_resource_readers( + resolve, timeout_seconds=0.05, worker_pool=pool + ) + + async def pool_occupied() -> None: + deadline = asyncio.get_running_loop().time() + 10 + while not pool.in_flight: + assert asyncio.get_running_loop().time() < deadline + await asyncio.sleep(0.01) + + occupant = None + if contention == "full_pool": + occupant = asyncio.create_task(pool.run(lambda: release.wait(timeout=30))) + await pool_occupied() + if service == "memory_preflight": + call = BasicPreflightService(reader=_guidance_reader(readers)).guidance( + workspace, "memory_store", {}, "Plan a decision." + ) + else: + call = BasicBriefingService(reader=_briefing_reader(readers)).assemble( + workspace, SessionBriefInput(workspace_id=workspace.workspace_id) + ) + task = asyncio.ensure_future(call) + try: + if contention == "overrun_read": + await pool_occupied() + await asyncio.sleep(0.2) + release.set() + with pytest.raises(RuntimeServiceError) as caught: + await task + finally: + release.set() + if occupant is not None: + await occupant + pool.shutdown() + assert caught.value.code == "DATABASE_IN_USE" + + +async def test_exclusive_storage_lock_is_database_in_use_not_unavailable(tmp_path): + """A migration/bootstrap holding the storage lock is transient contention.""" + from daem0nmcp.api.v7.runtime_services import WorkspaceStorageResolver + from daem0nmcp.storage_activation import DatabaseFileLock + + [workspace] = await initialize_workspaces((tmp_path,)) + holder = DatabaseFileLock(tmp_path / ".daem0nmcp" / "storage", "exclusive") + holder.acquire() + try: + with ( + pytest.raises(RuntimeServiceError) as caught, + WorkspaceStorageResolver().locked_active(workspace), + ): + pytest.fail("a shared read was admitted under an exclusive lock") + finally: + holder.release() + assert caught.value.code == "DATABASE_IN_USE" + with WorkspaceStorageResolver().locked_active(workspace) as active: + assert active.format_version == 7 + + +def test_internal_error_log_omits_pydantic_input_values(caplog): + from pydantic import BaseModel, ValidationError + + from daem0nmcp.api.v7.responses import ResponseFactory + + class Guidance(BaseModel): + priority: int + + # Built at runtime so the traceback's source lines cannot contain it. + stored_content = "-".join(("secret", "memory", "fragment", "canary")) + try: + try: + Guidance.model_validate({"priority": stored_content}) + except ValidationError as invalid: + raise RuntimeError("PREFLIGHT_FAILED") from invalid + except RuntimeError as error: + with caplog.at_level(logging.ERROR, logger="daem0nmcp.api.v7.responses"): + response = ResponseFactory().begin(None).internal_error(error) + + assert response.error.correlation_id in caplog.text + assert "priority (int_parsing)" in caplog.text + assert "PREFLIGHT_FAILED" in caplog.text + assert "test_internal_error_log_omits_pydantic_input_values" in caplog.text + assert "secret-memory-fragment-canary" not in caplog.text diff --git a/tests/api_v7/test_external_operations.py b/tests/api_v7/test_external_operations.py index 1a1126d..eda7561 100644 --- a/tests/api_v7/test_external_operations.py +++ b/tests/api_v7/test_external_operations.py @@ -737,6 +737,46 @@ async def test_missing_api_key_keeps_operation_registered_but_disabled( "Set E2B_API_KEY to enable isolated Python execution.", ) + async def test_disabling_the_profile_switches_remote_execution_off(self) -> None: + """The advertised opt-out must also stop the sandbox.""" + from daem0nmcp.api.v7.external_operations import ( + ExternalOperationDependencies, + ExternalOperationError, + build_external_operations, + ) + from daem0nmcp.api.v7.record_operations import RecordOperationDependencies + from daem0nmcp.workspace import WorkspaceRegistry + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + workspace = WorkspaceRegistry([root], default_root=root).default + operations = build_external_operations( + ExternalOperationDependencies( + record_dependencies=RecordOperationDependencies(), + environment={ + "E2B_API_KEY": "key", + "DAEM0NMCP_AGENCY_E2B_ENABLED": "false", + }, + sandbox_provider_factory=lambda: self.fail("provider was built"), + ) + ) + with self.assertRaises(ExternalOperationError) as caught: + await operations["sandbox_execute_python"]( + workspace=workspace, + request=_request( + "sandbox_execute_python", + workspace_id=workspace.workspace_id, + code="print(1)", + timeout_seconds=1, + preflight_token=PREFLIGHT_TOKEN, + ), + ) + + self.assertEqual(caught.exception.code, "CAPABILITY_DISABLED") + state = caught.exception.capability_states[0] + self.assertEqual(state.name, "agency-e2b") + self.assertIn("DAEM0NMCP_AGENCY_E2B_ENABLED", str(state.remediation)) + if __name__ == "__main__": unittest.main() diff --git a/tests/api_v7/test_graph_operations.py b/tests/api_v7/test_graph_operations.py index 12befa2..a8b7336 100644 --- a/tests/api_v7/test_graph_operations.py +++ b/tests/api_v7/test_graph_operations.py @@ -842,7 +842,12 @@ async def test_actual_mcp_graph_profile_enabled_and_disabled(tmp_path, transport "force": False, "idempotency_key": "process-graph-disabled", } - async with process_client(workspace.root, transport) as session: + # The graph extra is installed in CI, so it is on unless explicitly disabled. + async with process_client( + workspace.root, + transport, + environment_overrides={"DAEM0NMCP_GRAPH_ENABLED": "false"}, + ) as session: await succeed(session, "session_brief", scope) token = await _preflight( session, workspace.workspace_id, "entity_backfill", disabled_arguments diff --git a/tests/api_v7/test_models.py b/tests/api_v7/test_models.py index 17b4a9d..4d2e78a 100644 --- a/tests/api_v7/test_models.py +++ b/tests/api_v7/test_models.py @@ -70,6 +70,7 @@ def test_error_registry_is_exact_and_rejects_unknown_codes(self) -> None: "STALE_PROJECTION_ID", "CAPABILITY_DISABLED", "CAPABILITY_DEGRADED", + "MIGRATION_REQUIRED", "LEXICAL_UNAVAILABLE", "COMMUNION_REQUIRED", "COUNSEL_REQUIRED", @@ -247,33 +248,36 @@ def test_json_values_are_strict_finite_bounded_and_duplicate_safe(self) -> None: '{"tool":"memory_store","arguments":{"x":NaN}}' ) - def test_every_wire_model_rejects_absolute_paths_in_free_text_and_json( - self, - ) -> None: + def test_wire_models_reject_absolute_paths_outside_user_text(self) -> None: _, models = _load(self) + # A remedy echoes the caller's own arguments, which may mention paths. + models.ErrorRemedy( + tool="memory_store", + arguments={"context": {"note": "Inspect C:\\private\\secret.txt"}}, + ) with self.assertRaises(ValidationError): - models.ErrorRemedy( - tool="memory_store", - arguments={ - "context": { - "note": "Inspect C:\\private\\secret.txt before release" - } - }, + models.CapabilityState( + name="storage", + status="failed", + remediation="Repair C:\\private\\storage", ) + summary = { + "record_id": RECORD_ID, + "record_type": "warning", + "excerpt": "Do not read /home/private/.ssh/id_rsa", + "tags": ["C:\\proj"], + "relative_file_path": None, + "current_status": "current", + "content_hash": "a" * 64, + "created_at": NOW, + "updated_at": NOW, + } + # Excerpts and tags are user text (UD-3); the path field is not. + models.RecordSummary(**summary) with self.assertRaises(ValidationError): - models.RecordSummary( - record_id=RECORD_ID, - record_type="warning", - excerpt="Do not read /home/private/.ssh/id_rsa", - tags=[], - relative_file_path=None, - current_status="current", - content_hash="a" * 64, - created_at=NOW, - updated_at=NOW, - ) + models.RecordSummary(**{**summary, "relative_file_path": "/home/x"}) class EnvelopeTests(unittest.TestCase): diff --git a/tests/api_v7/test_operations.py b/tests/api_v7/test_operations.py index b042914..becdf49 100644 --- a/tests/api_v7/test_operations.py +++ b/tests/api_v7/test_operations.py @@ -755,23 +755,25 @@ def active_connection(*_args, **_kwargs): self.assertEqual(10_001, bundle.manifest["event_count"]) snapshot_builder.assert_called_once() - async def test_export_rejects_nested_absolute_path_strings(self) -> None: - secret = str(self.fixture.root / "private" / "nested-secret.txt") + async def test_export_carries_user_text_that_mentions_paths(self) -> None: + # Context is user text (UD-3); only the raw path columns are refused. + note = "Crash fixed in C:\\proj\\src\\app.py; see /health" self.fixture.append( "nested path source", occurred_at_us=1_700_000_000_000_000, recorded_at_us=1_700_000_000_000_100, - context_note=secret, + context_note=note, ) - with self.assertRaises(Exception) as caught: - await self._operations()["workspace_export"]( - workspace=self.fixture.workspace, - request=_request("workspace_export", workspace_id=WORKSPACE_ID), - ) + bundle = await self._operations()["workspace_export"]( + workspace=self.fixture.workspace, + request=_request("workspace_export", workspace_id=WORKSPACE_ID), + ) - self.assertEqual("WORKSPACE_PATH_ESCAPE", caught.exception.code) - self.assertNotIn(secret, str(caught.exception)) + self.assertEqual( + {"note": note}, + bundle.events[0].payload["data"]["record"]["context"], + ) async def test_cancelled_sqlite_work_propagates_and_finishes_privately( self, diff --git a/tests/api_v7/test_path_policy.py b/tests/api_v7/test_path_policy.py new file mode 100644 index 0000000..e68af7c --- /dev/null +++ b/tests/api_v7/test_path_policy.py @@ -0,0 +1,953 @@ +"""Path policy on the v7 wire (KD-2, UD-3). + +User text (memory content, rule text, queries, and server text quoting them) +may mention routes and file paths. Every other string keeps the +absolute-path rule, input and output, through one predicate. Anything the +server accepts must read back: nothing can be stored and then brick a read. +""" + +from __future__ import annotations + +import asyncio +import copy +import hashlib +import sqlite3 +import time +import types +from contextlib import closing +from pathlib import Path +from typing import Annotated, Any, Literal, Union, get_args, get_origin, get_type_hints + +import pytest +from fastmcp import Client +from pydantic import BaseModel, ValidationError, create_model + +from daem0nmcp.api.v7.errors import ErrorCode +from daem0nmcp.api.v7.models import ( + Cursor, + JsonObject, + RelativePath, + WireModel, + _user_text_fields, + is_host_absolute_path, +) +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resources import ( + ActiveContextResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + WarningResourceDocument, +) +from daem0nmcp.api.v7.responses import ResponseFactory +from daem0nmcp.api.v7.tools import ( + TOOL_DATA_MODELS, + TOOL_INPUT_MODELS, + CodeIndexInput, + DiagnosticSummary, + DocumentIngestUrlInput, + ExportEvent, + HealthData, + HttpsUrl, + MediumText, + MemoryStoreInput, + PortableVectorPoint, + SandboxExecutionData, +) +from daem0nmcp.config import Settings +from daem0nmcp.discovery_projection import ( + CommunityProjectionSeed, + DiscoveryProjectionBuilder, + EntityProjectionSeed, + EntityRecordSeed, +) +from daem0nmcp.event_store import ( + EventCommand, + EventStore, + GovernanceEventCommand, + GovernanceEventStore, +) +from daem0nmcp.retrieval.runtime import await_projection_job_drains +from daem0nmcp.retrieval.specialized_projection import SpecializedProjectionBuilder +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import initialize_workspaces +from tests.api_v7.test_process_workspace_isolation_sweep import ( + CANARY, + CROSS_WORKSPACE_TOOLS, + PROFILE_ENVIRONMENT, + _code, + _is_protected, + _mint_tokens, + _seed_workspace_a, + _sweep_arguments, + _target_call, +) + +# The UD-3 examples: a regex escape, routes, and a Windows file location. +PATH_CONTENT = ("\\\\d+", "/health", "GET /api/v1/users", "C:\\proj\\src\\app.py") +# Mechanical-coverage seeds (R-26). +SEEDS = ("C:\\proj\\x.py", "/health", "\\d+", "\x1b[0m") +NEUTRAL = "plain neutral words" +# A path in text is the user's business; these outcomes mean it was refused. +# Readers turn a row that fails its output model into CAPABILITY_DEGRADED. +REFUSALS = frozenset( + { + "INVALID_ARGUMENT", + "INTERNAL_ERROR", + "WORKSPACE_PATH_ESCAPE", + "SCHEMA_REJECTED", + "CAPABILITY_DEGRADED", + } +) +RESOURCE_KINDS = ("warnings", "failures", "rules", "active-context") +TOKEN = "preflight-token-placeholder-0001" +WS = "ws_" + "0" * 24 + + +def _invoker(client: Client): + """Call a tool; a schema rejection by FastMCP becomes SCHEMA_REJECTED.""" + + async def invoke(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await client.call_tool(tool, arguments, raise_on_error=False) + if result.structured_content is None: + assert result.is_error, tool + return {"ok": False, "error": {"code": "SCHEMA_REJECTED"}} + return result.structured_content + + return invoke + + +# --------------------------------------------------------------------------- +# Model-level checks. + + +def _store_input(**overrides: Any) -> dict[str, Any]: + return { + "workspace_id": WS, + "record_type": "decision", + "content": "text", + "idempotency_key": "path-policy-0001", + "preflight_token": TOKEN, + **overrides, + } + + +@pytest.mark.parametrize("path", ["C:\\x", "/etc/x"]) +def test_relative_file_path_rejects_host_paths(path): + with pytest.raises(ValidationError): + MemoryStoreInput.model_validate(_store_input(relative_file_path=path)) + + +@pytest.mark.parametrize("text", [*PATH_CONTENT, *SEEDS]) +def test_user_text_input_fields_accept_paths(text): + validated = MemoryStoreInput.model_validate( + _store_input( + content=text, + rationale=text, + tags=[text], + context={"note": text, text: [text]}, + ) + ) + assert validated.content == text + + +def test_server_generated_fields_reject_host_paths(): + with pytest.raises(ValidationError): + DiagnosticSummary(code="CODE_INDEX_SKIPPED", message="C:\\x") + health = { + "package_version": "7.0.0", + "protocol_version": "2025-06-18", + "supported_transports": {"stdio"}, + "auth_mode": "loopback", + "task_support": {"name": "tasks", "status": "ready"}, + } + HealthData.model_validate(health) + with pytest.raises(ValidationError): + HealthData.model_validate( + { + **health, + "capability_states": [ + { + "name": "storage", + "status": "failed", + "remediation": "Repair C:\\x", + } + ], + } + ) + + +def test_relative_paths_are_checked_by_their_own_validator(): + """Valid relative names and globs are not mistaken for absolute paths.""" + names = ["src/**/foo.py", "src/__pycache__/x", "docs-/a", "pkg_/a.py"] + index = CodeIndexInput(workspace_id=WS, patterns=names) + assert index.patterns == names + for invalid in ("C:\\x", "/etc/x", "../x", "src/../x", "~/x"): + with pytest.raises(ValidationError): + CodeIndexInput(workspace_id=WS, patterns=[invalid]) + + +def test_ingest_topic_accepts_paths(): + for text in SEEDS: + DocumentIngestUrlInput( + workspace_id=WS, + url="https://example.com/doc", + topic=text, + idempotency_key="path-policy-0001", + preflight_token=TOKEN, + ) + + +def test_error_remedies_echo_path_bearing_arguments(): + """A token error on path-bearing input returns its structured remedy (R-1).""" + for text in SEEDS: + failure = ( + ResponseFactory() + .begin(WS) + .failure( + ErrorCode.COUNSEL_REQUIRED, + "A preflight is required.", + remedy_tool="memory_preflight", + remedy_arguments={ + "workspace_id": WS, + "target_tool": "rule_create", + "target_arguments": {"trigger": text, "must_do": [text]}, + }, + ) + ) + assert failure.error.remedy.arguments["target_arguments"]["trigger"] == text + + +def test_sandbox_output_lines_may_mention_paths(): + line = 'File "/tmp/run.py", line 1, in ' + data = SandboxExecutionData( + success=False, + stdout="", + stderr=line, + exit_status=1, + execution_time_ms=5, + sanitized_logs=[f"stderr: {line}", "x" * 4096], + ) + assert data.sanitized_logs[0].endswith(line) + + +def test_export_structured_fields_stay_guarded(): + """Only user text in export payloads is exempt (R-3, R-4).""" + from daem0nmcp.api.v7.operations import CoreOperationError, _reject_raw_paths + + user = {"record": {"content": "/health", "context": {"note": "C:\\x"}}} + _reject_raw_paths([user]) + for relative in ("../../Users/x/y.py", "d:/other/x.py", "/etc/x"): + with pytest.raises(CoreOperationError): + _reject_raw_paths([{"record": {"file_path_relative": relative}}]) + with pytest.raises(CoreOperationError): + _reject_raw_paths([{"record": {"file_path": "C:\\proj\\x.py"}}]) + # A migrated event carries its v6 row as ``[name, value]`` pairs, so the + # guard checks that shape too -- without mistaking an ordinary two-element + # list for one, whose members may be unhashable. + _reject_raw_paths({"legacy": {"columns": [["keywords", "x"], ["id", 1]]}}) + _reject_raw_paths({"legacy": {"columns": [[1, 2], [3, 4]]}}) + _reject_raw_paths([[{"a": 1}, {"b": 2}]]) + for pair in ( + ["file_path", "C:\\proj\\x.py"], + ["file_path_relative", "../outside/x.py"], + ): + with pytest.raises(CoreOperationError): + _reject_raw_paths({"legacy": {"columns": [pair]}}) + event = { + "event_id": "evt_" + "2" * 64, + "event_type": "memory.created", + "happened_at": "2026-08-08T12:00:00Z", + "content_hash": "a" * 64, + "payload": {"data": {"content": "C:\\proj\\x.py"}, "actor_id": "agent"}, + } + ExportEvent.model_validate(event) + with pytest.raises(ValidationError): + ExportEvent.model_validate( + {**event, "payload": {**event["payload"], "actor_id": "C:\\proj"}} + ) + with pytest.raises(ValidationError): + PortableVectorPoint(point_id="p", payload={"model_id": "C:\\m"}, vector=[1.0]) + + +def test_every_input_field_shape_is_classified(): + """Discovery walks every input model, bundles included, and knows each shape.""" + paths = { + f"{tool}.{'.'.join(map(str, path))}" + for tool, model in TOOL_INPUT_MODELS.items() + for path, _kind_ in _free_text_paths(model) + } + assert { + "workspace_import.bundle.events.0.payload", + "workspace_import.bundle.legacy_rows", + "memory_store_batch.records.0.context", + } <= paths, sorted(paths) + + +def _models(root: type[BaseModel], seen: set[type[BaseModel]]) -> None: + if root in seen: + return + seen.add(root) + for field in root.model_fields.values(): + for model in _nested_models(field.annotation): + _models(model, seen) + + +def _nested_models(annotation: object): + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + yield annotation + for argument in get_args(annotation): + yield from _nested_models(argument) + + +def _sample(annotation: object, text: str) -> object: + """A value of ``annotation``'s shape whose every string is ``text``.""" + origin = get_origin(annotation) + if origin is Annotated: + return _sample(get_args(annotation)[0], text) + if origin in (Union, types.UnionType): + options = [item for item in get_args(annotation) if item is not type(None)] + return _sample(options[0], text) + if origin in (list, set, frozenset): + return [_sample(get_args(annotation)[0], text)] + if origin is dict: + return {text: _sample(get_args(annotation)[1], text)} + if annotation is JsonObject: + return {"note": text, text: [text]} + assert annotation is str, annotation + return text + + +def test_user_text_round_trips_into_every_output_field_that_carries_it(): + """Each string a UserText input accepts validates in every UserText output. + + This shows the exemption works for every output field already typed + ``UserText`` (found from the models, each probe inheriting ``WireModel``'s + path check). It cannot show that the right fields carry the type; the + end-to-end test below does that. + """ + outputs: set[type[BaseModel]] = set() + for model in ( + *TOOL_DATA_MODELS.values(), + WarningResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + ActiveContextResourceDocument, + ): + _models(model, outputs) + carriers = { + (model.__name__, name): Annotated[ + (field.annotation, *field.metadata) # type: ignore[valid-type] + ] + if field.metadata + else field.annotation + for model in outputs + for name, field in model.model_fields.items() + if name in _user_text_fields(model) + } + assert len(carriers) > 30, sorted(carriers) + for text in (*PATH_CONTENT, *SEEDS): + MemoryStoreInput.model_validate(_store_input(content=text)) + for (model_name, name), annotation in carriers.items(): + probe = create_model( + f"{model_name}_{name}", __base__=WireModel, value=(annotation, ...) + ) + probe.model_validate({"value": _sample(annotation, text)}) + # The probe is meaningful: a guarded text field refuses the same text. + guarded = create_model("Guarded", __base__=WireModel, value=(MediumText, ...)) + if is_host_absolute_path(text): + with pytest.raises(ValidationError): + guarded.model_validate({"value": text}) + + +# --------------------------------------------------------------------------- +# Rows as a migrated or pre-fix database carries them. + + +def _hex(*parts: object) -> str: + return hashlib.sha256(repr(parts).encode()).hexdigest() + + +def _now_us() -> int: + return time.time_ns() // 1_000 + + +def _memory_record(record_type: str, text: str) -> dict[str, Any]: + return { + "record_type": record_type, + "legacy_type": None, + "content": f"Migrated note: {text}", + "rationale": text, + "context": {"note": text}, + "tags": [text], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "migration", + "source_model": None, + "deleted_at_us": None, + } + + +def _insert_rows(root: Path, workspace_id: str, texts: tuple[str, ...]) -> list[str]: + """Write path-bearing records, a rule, a trigger, entities and a community.""" + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + record_ids: list[str] = [] + with closing(sqlite3.connect(database)) as connection: + connection.execute("PRAGMA foreign_keys=ON") + now = _now_us() - 1_000_000 + for index, text in enumerate(texts): + for record_type in ("decision", "warning"): + record_id = "mem_" + _hex("row", text, record_type) + EventStore(connection).append_and_project( + EventCommand( + workspace_id=workspace_id, + stream_id=record_id, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=now + index, + recorded_at_us=now + index, + actor_type="migration", + payload={"record": _memory_record(record_type, text)}, + ) + ) + record_ids.append(record_id) + GovernanceEventStore(connection).append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id="rule_" + _hex("rule", text), + stream_kind="rule", + event_type="rule.created", + occurred_at_us=now, + recorded_at_us=now, + actor_type="migration", + payload={ + "rule_id": "rule_" + _hex("rule", text), + "trigger": text, + "must_do": [text], + "must_not": [text], + "ask_first": [text], + "warnings": [text], + "priority": 0, + "enabled": True, + "created_at_us": now, + "updated_at_us": now, + }, + ) + ) + GovernanceEventStore(connection).append_and_project( + GovernanceEventCommand( + workspace_id=workspace_id, + stream_id="trg_" + _hex("trigger", text), + stream_kind="trigger", + event_type="context_trigger.created", + occurred_at_us=now, + recorded_at_us=now, + actor_type="migration", + payload={ + "trigger_id": "trg_" + _hex("trigger", text), + # A file glob: a tag pattern is a regex, which + # "C:\proj\x.py" is not. + "trigger_type": "file", + "pattern": text, + "recall_query": text, + "categories": [], + "enabled": True, + "priority": 0, + "created_at_us": now, + "updated_at_us": now, + "deleted_at_us": None, + }, + ) + ) + connection.commit() + SpecializedProjectionBuilder(connection).rebuild(workspace_id, "graph") + DiscoveryProjectionBuilder(connection).populate_graph( + workspace_id, + entities=tuple( + EntityProjectionSeed( + name=text, + entity_type="file", + records=(EntityRecordSeed(record_ids[2 * index]),), + ) + for index, text in enumerate(texts) + if is_host_absolute_path(text) + ), + communities=( + CommunityProjectionSeed( + source_key="paths", + label=texts[0], + level=0, + member_record_ids=tuple(record_ids), + ), + ), + ) + # The graph rebuild queued by the events above just ran here; retire + # it so a later drain cannot swap the inserted entities for a new + # generation mid-test. + connection.execute( + "UPDATE background_jobs SET status='succeeded',updated_at_us=?," + "finished_at_us=? WHERE workspace_id=? AND status='queued' " + "AND idempotency_key='active-projection:graph'", + (_now_us(), _now_us(), workspace_id), + ) + connection.commit() + return record_ids + + +def _server(roots: tuple[Path, ...]): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(roots[0]), + workspace_roots=[str(root) for root in roots], + dream_enabled=False, + ), + environ=PROFILE_ENVIRONMENT, + ) + + +async def test_path_content_is_stored_and_read_back(tmp_path): + """UD-3 content stores once and every read of it succeeds, pre-fix rows too.""" + root = tmp_path / "project" + (workspace,) = await initialize_workspaces((root,)) + ws = workspace.workspace_id + poisoned = _insert_rows(root, ws, ("\\\\d+",)) + async with Client(_server((root,))) as client: + invoke = _invoker(client) + + async def ok(tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + result = await _target_call(invoke, ws, tool, arguments) + assert result["ok"], (tool, arguments, result["error"]) + return result["data"] + + brief = await ok("session_brief", {}) + assert poisoned[1] in {item["record_id"] for item in brief["warnings"]} + for index, text in enumerate(PATH_CONTENT): + for record_type in ("decision", "warning"): + stored = await ok( + "memory_store", + { + "record_type": record_type, + "content": f"Remember {text}", + "idempotency_key": f"path-content-{index}-{record_type}", + }, + ) + record_id = stored["record"]["record_id"] + assert stored["record"]["excerpt"] == f"Remember {text}" + brief = await ok("session_brief", {}) + if record_type == "warning": + assert record_id in { + item["record_id"] for item in brief["warnings"] + } + await ok("memory_recall", {"query": text}) + await ok("memory_search_text", {"query": text}) + preflight = await invoke( + "memory_preflight", + { + "workspace_id": ws, + "target_tool": "memory_store", + "target_arguments": { + "record_type": record_type, + "content": text, + "idempotency_key": f"path-preflight-{index}", + }, + "description": text, + }, + ) + assert preflight["ok"], preflight["error"] + await ok( + "memory_record_outcome", + { + "record_id": record_id, + "outcome_text": text, + "worked": False, + "idempotency_key": f"path-outcome-{index}-{record_type}", + }, + ) + stale = await invoke( + "memory_store", + { + "workspace_id": ws, + "record_type": "decision", + "content": "GET /health returns 200", + "idempotency_key": "path-stale-token-0001", + "preflight_token": TOKEN, + }, + ) + assert stale["error"]["code"].startswith("TOKEN_"), stale + remedy = stale["error"]["remedy"] + assert remedy["tool"] == "memory_preflight" + assert remedy["arguments"]["target_arguments"]["content"] == ( + "GET /health returns 200" + ) + await ok("session_brief", {}) + for kind in RESOURCE_KINDS: + await client.read_resource(f"memory://workspaces/{ws}/{kind}") + + +# --------------------------------------------------------------------------- +# Mechanical coverage (R-26). + +# An import bundle is hash-chained: editing any one string makes it +# IMPORT_INVALID. Its fields are covered by exporting the seeded workspace +# and importing it back (see the end of the test), not by field seeding. +ROUND_TRIPPED = frozenset({"workspace_import"}) +# The ingest URL is fetched before the topic is read, and tests have no +# network, so every call stops at the URL. The topic's input type is +# covered by test_ingest_topic_accepts_paths. +FETCH_FIRST = frozenset({"document_ingest_url.topic"}) + + +def _mentions_str(annotation: object) -> bool: + return annotation is str or any(_mentions_str(a) for a in get_args(annotation)) + + +def _kind(annotation: object) -> object: + """``"str"``, ``"json"``, a model class, ``("list", kind)`` or ``None``. + + Fails on any string-bearing shape it cannot classify, so a new free-text + shape cannot silently drop out of coverage. + """ + if annotation in (Cursor, RelativePath, HttpsUrl): + return None + origin = get_origin(annotation) + if origin in (Union, types.UnionType): + kinds = {_kind(item) for item in get_args(annotation) if item is not type(None)} + kinds.discard(None) + assert len(kinds) <= 1, f"mixed union {annotation!r}" + return kinds.pop() if kinds else None + if origin is Annotated: + base, *metadata = get_args(annotation) + if any(getattr(item, "pattern", None) for item in metadata): + return None + return _kind(base) + if origin in (list, set, frozenset): + inner = _kind(get_args(annotation)[0]) + assert not isinstance(inner, tuple), f"nested list {annotation!r}" + return None if inner is None else ("list", inner) + if origin is Literal: + return None + if annotation is str: + return "str" + if annotation is JsonObject: + return "json" + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation + assert not _mentions_str(annotation), f"unclassified {annotation!r}" + return None + + +def _free_text_paths(model: type[BaseModel], prefix=()): + """Every free-text field of ``model`` and its nested models, by annotation.""" + hints = get_type_hints(model, include_extras=True) + for name in model.model_fields: + kind = _kind(hints[name]) + if kind is None: + continue + if isinstance(kind, type): + yield from _free_text_paths(kind, (*prefix, name)) + elif isinstance(kind, tuple) and isinstance(kind[1], type): + yield from _free_text_paths(kind[1], (*prefix, name, 0)) + else: + yield (*prefix, name), kind + + +def _seed_value(kind: object, text: str) -> object: + if kind == "str": + return text + if kind == "json": + return {"note": text, text: [text]} + if kind == ("list", "str"): + return [text] + assert kind == ("list", "json"), kind + return [{"note": text, text: [text]}] + + +def _seeded(tool: str, base: dict[str, Any], path: tuple, kind: object, text: str): + """``base`` with the field at ``path`` set to ``text``, and its siblings fit.""" + arguments = copy.deepcopy(base) + target: Any = arguments + for step, following in zip(path[:-1], path[1:], strict=True): + if isinstance(step, int): + while len(target) <= step: + target.append({}) + elif step not in target: + target[step] = [] if isinstance(following, int) else {} + target = target[step] + name = path[-1] + # Exactly one selector is allowed; the seeded name replaces the id. + target.pop( + {"entity_name": "entity_id", "qualified_name": "code_entity_id"}.get(name, ""), + None, + ) + if name == "procedure_steps": + target["record_type"] = "procedure" + if (tool, path) == ("memory_preflight", ("target_arguments",)): + # Seed the target tool's own argument, not the envelope. + target["target_tool"] = "memory_store" + target["target_arguments"] = { + "record_type": "decision", + "content": text, + "idempotency_key": "mech-preflight-target-0001", + } + return arguments + target[name] = _seed_value(kind, text) + return arguments + + +def _fresh_keys(arguments: dict[str, Any], label: str) -> dict[str, Any]: + if "idempotency_key" in arguments: + digest = hashlib.sha256(label.encode()).hexdigest()[:40] + return {**arguments, "idempotency_key": f"mech-{digest}"} + return arguments + + +def _own_read_arguments(arguments: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in arguments.items() + if key not in {"cursor", "after_cursor"} + } + + +async def _seed_free_text(invoke, workspace_id: str, tool: str, arguments: dict): + """Call ``tool`` with each free-text field set to each seed. + + The neutral-text call must not be refused, so a wrongly guarded field + cannot hide behind bad base arguments. (It may miss: a lookup by a name + that does not exist is NOT_FOUND.) Each seeded call must then get the + neutral call's outcome or at least not a refusal. + """ + if tool in ROUND_TRIPPED: + return [] + base = _own_read_arguments(arguments) + labels = [] + for path, kind in _free_text_paths(TOOL_INPUT_MODELS[tool]): + label = f"{tool}.{'.'.join(map(str, path))}" + if label in FETCH_FIRST: + continue + labels.append(label) + codes = [] + for text in (NEUTRAL, *SEEDS): + call = _fresh_keys(_seeded(tool, base, path, kind, text), label + text) + response = await _target_call(invoke, workspace_id, tool, call) + if _code(response) == "INTERNAL_ERROR": + # SQLite contention with a background projection drain is still + # reported as INTERNAL_ERROR on this base (PR 3 makes it + # retryable). A refusal caused by the text repeats; retry once. + call = _fresh_keys(call, label + text + "retry") + response = await _target_call(invoke, workspace_id, tool, call) + codes.append(_code(response)) + neutral, *seeded = codes + assert neutral not in REFUSALS, (label, neutral) + for text, code in zip(SEEDS, seeded, strict=True): + assert code == neutral or code not in REFUSALS, (label, text, code, neutral) + return labels + + +def _is_write(tool: str) -> bool: + # memory_record_outcome writes a memory event without a preflight; like + # every write it marks the graph stale until the next rebuild. + return _is_protected(tool) or tool == "memory_record_outcome" + + +async def _read_everything(invoke, workspace_id: str, seed) -> dict[str, str]: + """Every read tool with the workspace's own objects.""" + codes: dict[str, str] = {} + for tool, arguments in sorted(_sweep_arguments(seed).items()): + if _is_write(tool) or tool in CROSS_WORKSPACE_TOOLS: + continue + plain = _fresh_keys(_own_read_arguments(arguments), f"{tool}-{len(codes)}") + codes[tool] = _code(await invoke(tool, {"workspace_id": workspace_id, **plain})) + return codes + + +async def _export_pages(invoke, workspace_id: str) -> list[dict[str, Any]]: + scope = {"workspace_id": workspace_id, "page_byte_limit": 65536} + page = await invoke("workspace_export", scope) + assert page["ok"], page["error"] + pages = [page["data"]] + while not pages[-1]["complete"]: + previous = pages[-1] + page = await invoke( + "workspace_export", + { + **scope, + "export_session_id": previous["export_session_id"], + "page_index": previous["page_index"] + 1, + "cursor": previous["next_cursor"], + }, + ) + assert page["ok"], page["error"] + pages.append(page["data"]) + return pages + + +async def _import_call(invoke, workspace_id: str, arguments: dict) -> dict[str, Any]: + """Preflight and run one import call, waiting out a held write lock. + + The graph profile's projection drains can hold the SQLite write lock past + the server's busy timeout, which the preflight and the import report as the + retryable DATABASE_IN_USE. A real client retries those; so does this test, + within a bounded deadline and a bounded number of attempts. + """ + + deadline = time.monotonic() + 30 + for _attempt in range(8): + result = await _target_call(invoke, workspace_id, "workspace_import", arguments) + if result["ok"] or result["error"]["code"] != "DATABASE_IN_USE": + return result + assert time.monotonic() < deadline, result["error"] + await asyncio.sleep(0.5) + raise AssertionError("the import stayed contended for 8 attempts") + + +async def _import_pages(invoke, workspace_id: str, pages) -> dict[str, Any]: + import_id = None + key = "mech-import-0001" + for page in pages: + arguments = {"bundle": page, "finalize": False, "idempotency_key": key} + if import_id is not None: + arguments["import_session_id"] = import_id + staged = await _import_call(invoke, workspace_id, arguments) + assert staged["ok"], staged["error"] + import_id = staged["data"]["import_session_id"] + finish = {"import_session_id": import_id, "finalize": True, "idempotency_key": key} + done = await _import_call(invoke, workspace_id, finish) + assert done["ok"], done["error"] + return done["data"] + + +def _texts(value: Any) -> set[str]: + if isinstance(value, dict): + return {text for item in value.values() for text in _texts(item)} + if isinstance(value, list): + return {text for item in value for text in _texts(item)} + return {value} if isinstance(value, str) else set() + + +# Reads that stay gated whatever is stored; every other read must be ok. +GATED_READS = { + # Needs a native edit request, which only the edit hook makes. + "edit_preflight": "NOT_FOUND", + # The default selector exceeds the foreground node budget; the same + # tools are read below with a small selector over the path records. + "knowledge_graph_get": "TASK_REQUIRED", + "knowledge_graph_render": "TASK_REQUIRED", +} + + +async def test_every_free_text_field_accepts_paths_and_every_read_survives(tmp_path): + """Seed every free-text input field and DB row with paths; read everything. + + An input field typed as free text but not ``UserText`` refuses the seed, + and an output field carrying it fails or degrades the read; either fails + this test. The workspace is then exported and imported back. + """ + a_root, b_root = tmp_path / "alpha", tmp_path / "beta" + alpha, beta = await initialize_workspaces((a_root, b_root)) + a, b = alpha.workspace_id, beta.workspace_id + (a_root / "src").mkdir() + (a_root / "src" / "owned.py").write_text( + f"def {CANARY}_function():\n return 1 # TODO: see C:\\proj\\x.py /health\n", + encoding="utf-8", + ) + seeded: list[str] = [] + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + seed = await _seed_workspace_a(invoke, a, b) + await _mint_tokens(invoke, a, b, seed) + # Writes first; reads are seeded below, once every kind of row exists + # (a name lookup before any entity exists stops at the graph gate). + writes = { + tool: arguments + for tool, arguments in _sweep_arguments(seed).items() + if _is_write(tool) + } + writes["workspace_link"] = {"linked_workspace_id": b} + for tool, arguments in sorted(writes.items()): + seeded += await _seed_free_text(invoke, a, tool, arguments) + + # Rows as a migrated v6 database or a pre-fix write left them. + record_ids = _insert_rows(a_root, a, SEEDS) + + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + assert (await invoke("session_brief", {"workspace_id": a}))["ok"] + await _mint_tokens(invoke, a, b, seed) + entities = await invoke("entity_list", {"workspace_id": a, "limit": 100}) + path_seeds = {text for text in SEEDS if is_host_absolute_path(text)} + assert path_seeds <= {item["name"] for item in entities["data"]["items"]} + seed.ids["entity"] = entities["data"]["items"][0]["entity_id"] + # The inserted graph generation replaces the seed's community IDs. + communities = await invoke("community_list", {"workspace_id": a}) + seed.ids["community"] = communities["data"]["items"][0]["community_id"] + # The stored text comes back verbatim, not dropped or degraded. + # Before any read that writes (outcomes, rebuilds) makes the graph stale. + shown: set[str] = set() + for tool, arguments in ( + ("rule_list", {"limit": 100}), + ("context_trigger_list", {"limit": 100}), + ("memory_search_text", {"query": "Migrated note", "limit": 100}), + ( + "knowledge_graph_render", + {"record_ids": record_ids[:2], "include_orphans": True, "max_nodes": 2}, + ), + ( + "knowledge_graph_get", + {"record_ids": record_ids[:2], "include_orphans": True, "max_nodes": 2}, + ), + ): + response = await invoke(tool, {"workspace_id": a, **arguments}) + assert response["ok"], (tool, response["error"]) + shown |= _texts(response["data"]) + assert set(SEEDS) <= shown, set(SEEDS) - shown + assert {f"Migrated note: {text}" for text in SEEDS} <= shown + after = await _read_everything(invoke, a, seed) + assert {t: c for t, c in after.items() if c != "ok"} == GATED_READS, after + for tool, arguments in sorted(_sweep_arguments(seed).items()): + if tool in after: + # Query-like fields searching for the paths. + seeded += await _seed_free_text(invoke, a, tool, arguments) + for kind in RESOURCE_KINDS: + await client.read_resource(f"memory://workspaces/{a}/{kind}") + # Let background projection drains settle so the export pages form + # one consistent snapshot. + await await_projection_job_drains() + pages = await _export_pages(invoke, a) + + # The exported workspace imports back into empty storage (R-24). + storage = a_root / ".daem0nmcp" / "storage" + storage.rename(a_root / ".daem0nmcp" / "retained-before-import") + async with Client(_server((a_root, b_root))) as client: + invoke = _invoker(client) + assert (await invoke("session_brief", {"workspace_id": a}))["ok"] + restored = await _import_pages(invoke, a, pages) + assert restored["root_hash"] == pages[0]["root_hash"] + found = await invoke( + "memory_search_text", + {"workspace_id": a, "query": "Migrated note", "limit": 100}, + ) + assert found["ok"], found["error"] + assert {f"Migrated note: {text}" for text in SEEDS} <= _texts(found["data"]) + + # Nested and cross-workspace inputs are reached too; the floor catches a + # change that quietly drops fields from coverage. + assert { + "memory_store.context", + "memory_store.procedure_steps", + "memory_store_batch.records.0.content", + "memory_preflight.target_arguments", + "entity_evolution_trace.entity_name", + "rule_update.patch.must_do", + "workspace_link.label", + } <= set(seeded), seeded + assert len(seeded) >= 60, len(seeded) diff --git a/tests/api_v7/test_pinned_handlers.py b/tests/api_v7/test_pinned_handlers.py index 6900624..3bb5f98 100644 --- a/tests/api_v7/test_pinned_handlers.py +++ b/tests/api_v7/test_pinned_handlers.py @@ -109,13 +109,29 @@ async def guidance(self, *args: object, **kwargs: object) -> object: class _LeakyPreflightService: - async def guidance(self, *args: object, **kwargs: object) -> object: - from daem0nmcp.api.v7.tools import PreflightGuidance + """Guidance whose structured path field holds a host path. + + The rule warning also names a path, which user text may do (UD-3); only + ``relative_file_path`` makes the guidance invalid. + """ + async def guidance(self, *args: object, **kwargs: object) -> object: del args, kwargs - return PreflightGuidance( - warnings=[r"Inspect D:\private\workspace\policy.txt"], - ) + return { + "warnings": [r"Inspect D:\private\workspace\notes.txt"], + "records": [ + { + "record_id": "mem_" + "1" * 64, + "record_type": "warning", + "excerpt": "Guarded path field.", + "relative_file_path": r"D:\private\workspace\policy.txt", + "current_status": "current", + "content_hash": "a" * 64, + "created_at": "2026-08-08T12:00:00Z", + "updated_at": "2026-08-08T12:00:00Z", + } + ], + } class _ExplodingRecallService: @@ -1000,7 +1016,7 @@ def test_failed_preflight_guidance_is_opaque_and_issues_no_token(self) -> None: self.assertEqual(gate.state_store.status(scope)["active_capabilities"], 0) def test_path_bearing_guidance_fails_before_capability_issue(self) -> None: - # Catches issuing a usable token for a response that must be redacted. + # Catches issuing a usable token for guidance whose path field is invalid. from daem0nmcp.api.v7.errors import ErrorCode from daem0nmcp.api.v7.pinned import build_pinned_handlers from daem0nmcp.api.v7.tools import MemoryPreflightOutput @@ -1173,6 +1189,12 @@ async def guidance( self.assertTrue(validated.ok) self.assertIsNone(validated.data.preflight_token) self.assertIsNone(validated.data.expires_at) + # An ok:true draft must say why no token was issued (F-070/F-082). + [warning] = validated.meta.warnings + self.assertEqual("PREFLIGHT_TOKEN_NOT_ISSUED", warning.code) + self.assertIn("record_type (missing)", warning.message) + self.assertIn("content (missing)", warning.message) + self.assertIn("No preflight token issued", warning.message) self.assertEqual(len(observed), 1) self.assertEqual(observed[0][0], "memory_store") self.assertEqual(gate.state_store.status(scope)["active_capabilities"], 0) @@ -1761,6 +1783,56 @@ def test_record_outcome_uses_a_sanitized_task7_event_receipt(self) -> None: self.assertTrue(validated.data.worked) self.assertFalse(validated.data.idempotent_replay) + def test_transient_writer_contention_is_retryable_database_in_use(self) -> None: + # Pool saturation and SQLite lock timeouts clear on retry (F-020/F-027). + from daem0nmcp.api.v7 import pinned + from daem0nmcp.api.v7.errors import ErrorCode + from daem0nmcp.api.v7.runtime_services import RuntimeServiceError + from daem0nmcp.api.v7.tools import MemoryStoreInput, MemoryStoreOutput + + workspace = Workspace(WORKSPACE_ID, Path.cwd()) + scope = InvocationScope("principal", "session-busy", str(workspace.root)) + gate = _gate() + gate.record_briefing(scope) + target = { + "workspace_id": WORKSPACE_ID, + "record_type": "decision", + "content": "Contention is transient.", + "idempotency_key": "decision-busy-1", + } + token = gate.issue_preflight(scope, "memory_store", target) + handlers = pinned.build_pinned_handlers( + _dependencies( + gate=gate, + scope=scope, + workspace=workspace, + briefing_service=_UnusedService(), + memory_event_writer=_FailingMemoryEventWriter( + RuntimeServiceError("DATABASE_IN_USE") + ), + ) + ) + + request = MemoryStoreInput(**target, preflight_token=token).model_dump() + response = asyncio.run(handlers["memory_store"](**request)) + + validated = MemoryStoreOutput.model_validate(response) + self.assertFalse(validated.ok) + self.assertEqual(ErrorCode.DATABASE_IN_USE, validated.error.code) + self.assertTrue(validated.error.retryable) + self.assertEqual(250, validated.error.retry_after_ms) + # authorize() spent the token before the write, so the documented + # retry is a fresh preflight with the same idempotency_key. + self.assertEqual("memory_preflight", validated.error.remedy.tool) + remedy_target = validated.error.remedy.arguments["target_arguments"] + self.assertEqual("decision-busy-1", remedy_target["idempotency_key"]) + self.assertNotIn("preflight_token", remedy_target) + replayed = MemoryStoreOutput.model_validate( + asyncio.run(handlers["memory_store"](**request)) + ) + self.assertEqual(ErrorCode.TOKEN_REPLAYED, replayed.error.code) + self.assertEqual("memory_preflight", replayed.error.remedy.tool) + def test_record_outcome_maps_runtime_not_found(self) -> None: from daem0nmcp.api.v7 import pinned from daem0nmcp.api.v7.errors import ErrorCode diff --git a/tests/api_v7/test_portable_projections.py b/tests/api_v7/test_portable_projections.py index 0633ffc..7ab99fe 100644 --- a/tests/api_v7/test_portable_projections.py +++ b/tests/api_v7/test_portable_projections.py @@ -1995,3 +1995,115 @@ def blocking_prepare(*args, **kwargs): assert artifacts == 0 attempts = directory / "attempts" assert not attempts.exists() or not any(attempts.iterdir()) + + +def test_a_concurrent_writer_is_reported_as_retryable_not_as_a_bad_bundle(): + """A held SQLite write lock must not read as an invalid bundle.""" + from daem0nmcp.api.v7.operations import _portable_failure_code + from daem0nmcp.api.v7.portable_projections import PortableTransferError + + busy = sqlite3.OperationalError("database is locked") + wrapped = PortableTransferError("IMPORT_INVALID") + wrapped.__cause__ = busy + # A bad bundle raised while an unrelated busy error is in flight. + unrelated = PortableTransferError("IMPORT_INVALID") + unrelated.__context__ = busy + + assert _portable_failure_code(busy, "workspace_import") == "DATABASE_IN_USE" + assert _portable_failure_code(wrapped, "workspace_import") == "DATABASE_IN_USE" + assert _portable_failure_code(unrelated, "workspace_import") == "IMPORT_INVALID" + assert ( + _portable_failure_code(ValueError("bad page"), "workspace_import") + == "IMPORT_INVALID" + ) + + +async def test_retry_after_a_contended_finalize_releases_its_abandoned_lease( + tmp_path: Path, +): + """A finalize retried after DATABASE_IN_USE must not meet its own dead lease. + + The failed attempt claims a durable lease, then a concurrent writer holds the + SQLite write lock past the busy timeout, so both the attempt and its cleanup + release report "database is locked". The retry that DATABASE_IN_USE invites + must finish the import, not answer the non-retryable TASK_REQUIRED until the + abandoned lease expires. + """ + from daem0nmcp.api.v7 import operations as operations_module + + source = _fixture(tmp_path / "source") + target = _fixture(tmp_path / "target") + source.append("contended finalize", occurred_at_us=100, recorded_at_us=200) + pages = await _all_pages( + build_core_operations(_dependencies(source))["workspace_export"], + source, + include_legacy_projection=False, + ) + importer = build_core_operations(_dependencies(target))["workspace_import"] + session_id = None + for index, page in enumerate(pages): + staged = await importer( + workspace=target.workspace, + request=_request( + "workspace_import", + workspace_id=WORKSPACE_ID, + bundle=page.model_dump(mode="python"), + import_session_id=session_id, + finalize=False, + idempotency_key=f"contended-stage-{index:04d}", + preflight_token="token_value_for_test", + ), + ) + session_id = staged.import_session_id + finalize_request = _request( + "workspace_import", + workspace_id=WORKSPACE_ID, + bundle=None, + import_session_id=session_id, + finalize=True, + idempotency_key="contended-finalize-0001", + preflight_token="token_value_for_test", + ) + original_prepare = operations_module.prepare_vector_candidate + original_release = operations_module.release_import_finalization + calls = {"prepare": 0, "release": 0} + + def contended_prepare(*args, **kwargs): + calls["prepare"] += 1 + if calls["prepare"] == 1: + raise sqlite3.OperationalError("database is locked") + return original_prepare(*args, **kwargs) + + def contended_release(*args, **kwargs): + calls["release"] += 1 + if calls["release"] == 1: + raise sqlite3.OperationalError("database is locked") + return original_release(*args, **kwargs) + + with ( + patch.object( + operations_module, + "prepare_vector_candidate", + side_effect=contended_prepare, + ), + patch.object( + operations_module, + "release_import_finalization", + side_effect=contended_release, + ), + ): + with pytest.raises(CoreOperationError, match="DATABASE_IN_USE"): + await importer(workspace=target.workspace, request=finalize_request) + result = await importer(workspace=target.workspace, request=finalize_request) + assert result.status == "succeeded" + assert result.imported == 1 + connection = sqlite3.connect(target.database) + try: + leases = connection.execute( + "SELECT COUNT(*) FROM portable_transfer_finalization_leases " + "WHERE session_id=?", + (session_id,), + ).fetchone()[0] + finally: + connection.close() + assert leases == 0 diff --git a/tests/api_v7/test_process_client.py b/tests/api_v7/test_process_client.py index 8f71340..cb24ee5 100644 --- a/tests/api_v7/test_process_client.py +++ b/tests/api_v7/test_process_client.py @@ -151,13 +151,7 @@ async def test_production_ritual_and_restart(initialized_workspace, transport): "idempotency_key": "process-acceptance-outcome-0001", }, ) - visibility_deadline = asyncio.get_running_loop().time() + 5 - while True: - recalled = await succeed(session, "memory_recall", recall_arguments) - if record_id in {item["record"]["record_id"] for item in recalled["items"]}: - break - assert asyncio.get_running_loop().time() < visibility_deadline, recalled - await asyncio.sleep(0.1) + await _recall_until_visible(session, recall_arguments, record_id) templates = await session.list_resource_templates() assert len(templates.resourceTemplates) >= 4 resource = await session.read_resource( @@ -172,8 +166,24 @@ async def test_production_ritual_and_restart(initialized_workspace, transport): denied = await call(restarted, "memory_recall", recall_arguments) assert denied["error"]["code"] == "COMMUNION_REQUIRED" await succeed(restarted, "session_brief", scope) - recalled = await succeed(restarted, "memory_recall", recall_arguments) - assert record_id in {item["record"]["record_id"] for item in recalled["items"]} + # A restarted server republishes the lexical generation in the + # background, so the first recall after it can still read the + # previous one. Poll to a bounded deadline instead of racing it. + await _recall_until_visible(restarted, recall_arguments, record_id) + + +async def _recall_until_visible( + session, recall_arguments: dict, record_id: str, *, timeout: float = 30 +) -> None: + """Recall until the record is visible, or fail at a bounded deadline.""" + + deadline = asyncio.get_running_loop().time() + timeout + while True: + recalled = await succeed(session, "memory_recall", recall_arguments) + if record_id in {item["record"]["record_id"] for item in recalled["items"]}: + return + assert asyncio.get_running_loop().time() < deadline, recalled + await asyncio.sleep(0.1) async def test_federated_recall_process_requires_each_briefing_and_keeps_origin( @@ -452,3 +462,22 @@ async def test_consolidation_preview_copy_and_canonical_archive_over_real_mcp( ) assert archived["archived"] == 2 assert all(source.root.is_dir() for source in sources) + + +async def test_an_installed_extra_is_enabled_without_any_variable( + initialized_workspace, +): + """UD-7: installing the extra is the whole opt-in.""" + from daem0nmcp.capabilities import CapabilityRegistry + + if CapabilityRegistry(environ={}).get("graph")["status"] != "ready": + pytest.skip("the graph extra is not installed") + + workspace = initialized_workspace + scope = {"workspace_id": workspace.workspace_id} + async with process_client(workspace.root, "stdio") as session: + await succeed(session, "session_brief", scope) + health = await succeed(session, "system_health", scope) + + states = {state["name"]: state["status"] for state in health["capability_states"]} + assert states["graph"] == "ready", states diff --git a/tests/api_v7/test_process_code_impact.py b/tests/api_v7/test_process_code_impact.py index 4beb9dd..8e7f852 100644 --- a/tests/api_v7/test_process_code_impact.py +++ b/tests/api_v7/test_process_code_impact.py @@ -111,7 +111,11 @@ async def test_production_code_index_search_and_impact(code_workspace, transport async def test_production_code_index_disabled_profile_has_remedy(code_workspace): scope = {"workspace_id": code_workspace.workspace_id} - async with process_client(code_workspace.root, "stdio") as session: + async with process_client( + code_workspace.root, + "stdio", + environment_overrides={"DAEM0NMCP_APPS_ENABLED": "false"}, + ) as session: await succeed(session, "session_brief", scope) disabled = await call( session, @@ -122,4 +126,4 @@ async def test_production_code_index_disabled_profile_has_remedy(code_workspace) assert disabled["error"]["code"] == "CAPABILITY_DISABLED" states = disabled["meta"]["capability_states"] assert states[0]["name"] == "apps" - assert "DAEM0NMCP_APPS_ENABLED=true" in states[0]["remediation"] + assert "unless DAEM0NMCP_APPS_ENABLED=false" in states[0]["remediation"] diff --git a/tests/api_v7/test_process_concurrent_writers.py b/tests/api_v7/test_process_concurrent_writers.py new file mode 100644 index 0000000..dc7b88b --- /dev/null +++ b/tests/api_v7/test_process_concurrent_writers.py @@ -0,0 +1,117 @@ +"""Two stdio server processes writing one workspace store each key exactly once.""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from collections.abc import Callable +from contextlib import closing +from pathlib import Path +from typing import Any + +from mcp import ClientSession + +from daem0nmcp.storage_activation import resolve_active_database +from tests.api_v7.process_client import call, initialize_workspaces, process_client + +SAME_KEY_STORES = 5 +DISTINCT_KEY_STORES = 10 +LOG_NAMES = ("stdio-server-a.log", "stdio-server-b.log") + + +def _server_log_tails(root: Path, lines: int = 40) -> str: + """Server stderr (tracebacks by correlation ID) for assertion messages.""" + tails = [] + for name in LOG_NAMES: + try: + text = (root / name).read_text(encoding="utf-8", errors="replace") + except OSError as error: + text = f"" + tails.append(f"--- {name} ---\n" + "\n".join(text.splitlines()[-lines:])) + return "\n".join(tails) + + +async def _store_until_ok( + session: ClientSession, + workspace_id: str, + key: str, + content: str, + diagnostics: Callable[[], str], +) -> list[dict[str, Any]]: + """Preflight and store, retrying only what the server marks retryable.""" + scope = {"workspace_id": workspace_id} + arguments = {"record_type": "decision", "content": content, "idempotency_key": key} + observed: list[dict[str, Any]] = [] + deadline = asyncio.get_running_loop().time() + 120 + while True: + response = await call( + session, + "memory_preflight", + {**scope, "target_tool": "memory_store", "target_arguments": arguments}, + ) + if response["ok"]: + token = response["data"]["preflight_token"] + response = await call( + session, + "memory_store", + {**scope, **arguments, "preflight_token": token}, + ) + observed.append(response) + if response["ok"]: + return observed + retryable = ( + response["error"]["code"] == "DATABASE_IN_USE" + and response["error"]["retryable"] is True + ) + assert retryable, (observed, diagnostics()) + assert asyncio.get_running_loop().time() < deadline, (observed, diagnostics()) + await asyncio.sleep(response["error"]["retry_after_ms"] / 1000) + + +async def test_two_stdio_servers_store_each_key_exactly_once(tmp_path): + [workspace] = await initialize_workspaces((tmp_path,)) + scope = {"workspace_id": workspace.workspace_id} + async with ( + process_client(tmp_path, "stdio", log_name=LOG_NAMES[0]) as first, + process_client(tmp_path, "stdio", log_name=LOG_NAMES[1]) as second, + ): + sessions = (first, second) + for session in sessions: + assert (await call(session, "session_brief", scope))["ok"] + stores = [ + ("shared-store-key-0001", "One decision stored from two processes") + ] * SAME_KEY_STORES + [ + (f"distinct-store-key-{index:04d}", f"Distinct decision {index}") + for index in range(DISTINCT_KEY_STORES) + ] + results = await asyncio.gather( + *( + _store_until_ok( + sessions[index % 2], + workspace.workspace_id, + key, + content, + lambda: _server_log_tails(tmp_path), + ) + for index, (key, content) in enumerate(stores) + ) + ) + + # Logged, not asserted: a fast host may never contend. + print("DATABASE_IN_USE retries:", sum(len(observed) - 1 for observed in results)) + record_ids = {observed[-1]["data"]["record"]["record_id"] for observed in results} + assert len(record_ids) == 1 + DISTINCT_KEY_STORES + replays = [observed[-1]["data"]["idempotent_replay"] for observed in results] + assert replays[:SAME_KEY_STORES].count(False) == 1, replays + + database = resolve_active_database(tmp_path / ".daem0nmcp" / "storage").path + with closing(sqlite3.connect(database)) as connection: + per_stream = dict( + connection.execute( + "SELECT stream_id, count(*) FROM memory_events " + "WHERE event_type='memory.created' GROUP BY stream_id" + ).fetchall() + ) + records = connection.execute("SELECT count(*) FROM memory_records").fetchone() + assert per_stream == dict.fromkeys(record_ids, 1) + assert records[0] == 1 + DISTINCT_KEY_STORES diff --git a/tests/api_v7/test_process_discovery_surface.py b/tests/api_v7/test_process_discovery_surface.py index 2b3f101..60e9377 100644 --- a/tests/api_v7/test_process_discovery_surface.py +++ b/tests/api_v7/test_process_discovery_surface.py @@ -2,13 +2,14 @@ from __future__ import annotations +import asyncio import importlib.util import pytest from daem0nmcp.database import DatabaseManager from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed @pytest.fixture @@ -132,15 +133,24 @@ async def test_production_graph_discovery_surface(initialized_workspace, transpo {**scope, "entity_id": entity["entity_id"]}, ) assert traced["entity"]["entity_id"] == entity["entity_id"] - recalled = await succeed( - session, - "memory_recall_entity", - {**scope, "entity_name": entity["name"]}, - ) - assert {item["record_id"] for item in recalled["items"]} & { - first["record_id"], - second["record_id"], - } + # The lexical projection may still be catching up with the link and + # backfill events; recall reports that as retryable until it has. + recall_deadline = asyncio.get_running_loop().time() + 30 + while True: + recalled = await call( + session, + "memory_recall_entity", + {**scope, "entity_name": entity["name"]}, + ) + if recalled["ok"] and { + item["record_id"] for item in recalled["data"]["items"] + } & {first["record_id"], second["record_id"]}: + break + if not recalled["ok"]: + assert recalled["error"]["code"] == "DATABASE_IN_USE", recalled + assert recalled["error"]["retryable"], recalled + assert asyncio.get_running_loop().time() < recall_deadline, recalled + await asyncio.sleep(0.25) graph = await succeed( session, "knowledge_graph_get", diff --git a/tests/api_v7/test_process_dreaming.py b/tests/api_v7/test_process_dreaming.py index 8954f36..bc391a6 100644 --- a/tests/api_v7/test_process_dreaming.py +++ b/tests/api_v7/test_process_dreaming.py @@ -82,6 +82,8 @@ async def test_production_dreaming_runs_idempotently_and_reports_missing_graph( "DAEM0NMCP_DREAM_PENDING_MIN_AGE_HOURS": "0", "DAEM0NMCP_DREAM_PENDING_EVIDENCE_THRESHOLD": "1", "DAEM0NMCP_DREAM_REVIEW_COOLDOWN_HOURS": "72", + # CI installs the graph extra, which would otherwise turn graph on. + "DAEM0NMCP_GRAPH_ENABLED": "false", } scope = {"workspace_id": workspace.workspace_id} async with process_client( diff --git a/tests/api_v7/test_process_portability.py b/tests/api_v7/test_process_portability.py index 40e135d..f86cd8c 100644 --- a/tests/api_v7/test_process_portability.py +++ b/tests/api_v7/test_process_portability.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import sqlite3 from contextlib import closing from datetime import datetime, timedelta, timezone @@ -12,7 +13,7 @@ from daem0nmcp.database import DatabaseManager from daem0nmcp.event_store import EventCommand, EventStore from daem0nmcp.workspace import WorkspaceRegistry -from tests.api_v7.process_client import process_client, succeed +from tests.api_v7.process_client import call, process_client, succeed from tests.api_v7.test_process_surface import _preflight, _store @@ -40,6 +41,52 @@ async def _export_pages(session, scope, legacy): return pages +async def _import_retrying_while_busy(session, scope, arguments): + """Import one page, waiting out a concurrent writer's SQLite write lock. + + A projection drain or dreaming job can hold the write lock past the server's + busy timeout, which both the preflight and the transfer report as the + retryable DATABASE_IN_USE. A real client retries those; so does this test, + within a bounded deadline and a bounded number of attempts. + """ + + deadline = asyncio.get_running_loop().time() + 30 + for attempt in range(8): + contended = None + preflight = await call( + session, + "memory_preflight", + { + **scope, + "target_tool": "workspace_import", + "target_arguments": arguments, + "description": "Exercise workspace_import through the transport", + }, + ) + if preflight["ok"]: + result = await call( + session, + "workspace_import", + { + **scope, + **arguments, + "preflight_token": preflight["data"]["preflight_token"], + }, + ) + if result["ok"]: + # Contention is occasional; a transfer that always contends is + # a regression, not slowness. + assert attempt < 4, f"import needed {attempt + 1} attempts" + return result["data"] + contended = result["error"] + else: + contended = preflight["error"] + assert contended["code"] == "DATABASE_IN_USE", contended + assert asyncio.get_running_loop().time() < deadline, contended + await asyncio.sleep(0.5) + raise AssertionError("the import stayed contended for 8 attempts") + + @pytest.mark.parametrize("transport", ["stdio", "streamable-http"]) @pytest.mark.parametrize("legacy", [False, True]) async def test_production_export_restores_retained_workspace( @@ -78,7 +125,19 @@ async def test_production_export_restores_retained_workspace( retained = tmp_path / ".daem0nmcp" / "retained-before-restore" assert storage.resolve().is_relative_to(tmp_path.resolve()) assert retained.resolve().is_relative_to(tmp_path.resolve()) - storage.rename(retained) + # Windows can briefly refuse the rename after the server exits. The cause is + # unconfirmed (the venv launcher's real interpreter is a grandchild the + # harness does not wait for, or a virus scanner reading the fresh files); + # no product code renames storage. Retry within a bounded deadline. + rename_deadline = asyncio.get_running_loop().time() + 15 + while True: + try: + storage.rename(retained) + break + except PermissionError: + if asyncio.get_running_loop().time() >= rename_deadline: + raise + await asyncio.sleep(0.2) async with process_client(tmp_path, transport) as session: await succeed(session, "session_brief", scope) import_id = None @@ -90,17 +149,7 @@ async def test_production_export_restores_retained_workspace( } if import_id is not None: arguments["import_session_id"] = import_id - staged = await succeed( - session, - "workspace_import", - { - **scope, - **arguments, - "preflight_token": await _preflight( - session, scope, "workspace_import", arguments - ), - }, - ) + staged = await _import_retrying_while_busy(session, scope, arguments) import_id = staged["import_session_id"] assert staged["staged_pages"] == index + 1 assert staged["imported"] == 0 @@ -109,17 +158,7 @@ async def test_production_export_restores_retained_workspace( "finalize": True, "idempotency_key": "portable-process-import-0001", } - restored = await succeed( - session, - "workspace_import", - { - **scope, - **finish, - "preflight_token": await _preflight( - session, scope, "workspace_import", finish - ), - }, - ) + restored = await _import_retrying_while_busy(session, scope, finish) assert restored["status"] == "succeeded" assert restored["imported"] == 2 assert restored["root_hash"] == expected_root diff --git a/tests/api_v7/test_process_workspace_isolation_sweep.py b/tests/api_v7/test_process_workspace_isolation_sweep.py index 93ea35a..c180ca2 100644 --- a/tests/api_v7/test_process_workspace_isolation_sweep.py +++ b/tests/api_v7/test_process_workspace_isolation_sweep.py @@ -13,8 +13,11 @@ No target response may carry A's text, A's canary token, A's IDs or A's workspace ID; A's database may not change; a rejected call may not change the -target; and no string or key anywhere may carry a registered root or the home -directory. Every call's outcome is pinned. +target; and no string or key outside a ``UserText`` field may carry a +registered root or the home directory. Each target stores a note naming its +own root, which user text may do (UD-3), so the fields that echo it are found +from the same response models the server validates against, never by field +name. Every call's outcome is pinned. Pass 2 relaunches A and B over HTTP with a JWT principal granted only A. Every call that targets B, directly or as a consolidation or federation @@ -38,9 +41,18 @@ import pytest from fastmcp import Client from mcp.shared.exceptions import McpError +from pydantic import BaseModel, ValidationError +from daem0nmcp.api.v7.models import ApiResponse, RecordSummary, guarded_strings from daem0nmcp.api.v7.policy import V7_TOOL_LEVELS from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.api.v7.resources import ( + ActiveContextResourceDocument, + FailureResourceDocument, + RuleResourceDocument, + WarningResourceDocument, +) +from daem0nmcp.api.v7.tools import TOOL_DATA_MODELS, DiagnosticSummary from daem0nmcp.config import Settings from daem0nmcp.covenant import CovenantLevel from daem0nmcp.storage_activation import resolve_active_database @@ -94,6 +106,43 @@ Invoke = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] +RESOURCE_MODELS: dict[str, type[BaseModel]] = { + "warnings": WarningResourceDocument, + "failures": FailureResourceDocument, + "rules": RuleResourceDocument, + "active-context": ActiveContextResourceDocument, +} + + +def _response_model(tool: str) -> type[BaseModel]: + return ApiResponse[TOOL_DATA_MODELS[tool]] # type: ignore[name-defined] + + +# Marks the one piece of user text allowed to name a host path: the note each +# target stores about its own root. +NOTE_MARKER = "sweepnote4b1d" + + +def _host_path_leaks( + model: type[BaseModel], document: Any, needles: set[str] +) -> list[str]: + """Strings that contain a host path needle, other than the stored note. + + Every string is scanned, UserText fields included, so server-assembled + user-text fields (excerpts, rendered context, TODO text, mermaid) stay + watched. A hit is allowed only in a UserText field whose text carries the + note's marker. Validating against the response model also refuses any + absolute path in a server-generated field, whatever its spelling. + """ + validated = model.model_validate(document, strict=False) + guarded = set(guarded_strings(validated)) + return [ + value + for value in _strings(document) + if any(needle in value.lower() for needle in needles) + and (value in guarded or NOTE_MARKER not in value) + ] + def _placeholder(prefix: str) -> str: return f"{prefix}_{hashlib.sha256(prefix.encode()).hexdigest()}" @@ -641,6 +690,36 @@ def _is_protected(tool: str) -> bool: return V7_TOOL_LEVELS[tool] in {CovenantLevel.COUNSEL, CovenantLevel.DESTRUCTIVE} +def test_leak_check_exempts_user_text_by_annotation_only(tmp_path): + needles = {str(tmp_path).lower(), "alpha-private-root"} + summary = { + "record_id": "mem_" + "1" * 64, + "record_type": "warning", + "excerpt": f"Target note {NOTE_MARKER}: see {tmp_path}", + "tags": ["/health"], + "current_status": "current", + "content_hash": "a" * 64, + "created_at": "2026-08-08T12:00:00Z", + "updated_at": "2026-08-08T12:00:00Z", + } + assert _host_path_leaks(RecordSummary, summary, needles) == [] + # A root in user text the server assembled on its own is still caught. + unmarked = {**summary, "excerpt": f"Rendered from {tmp_path}"} + assert _host_path_leaks(RecordSummary, unmarked, needles) == [ + f"Rendered from {tmp_path}" + ] + # The same root in a server-generated field is refused by the model... + with pytest.raises(ValidationError): + _host_path_leaks( + DiagnosticSummary, {"code": "X_Y", "message": f"at {tmp_path}"}, needles + ) + # ...and a spelling the path rule does not see is found by the scan. + leaked = {"code": "X_Y", "message": "at alpha-private-root"} + assert _host_path_leaks(DiagnosticSummary, leaked, needles) == [ + "at alpha-private-root" + ] + + def test_every_manifest_tool_is_swept_or_declared_cross_workspace(): swept = set(_sweep_arguments(Seed())) cross = set(CROSS_WORKSPACE_TOOLS) @@ -666,7 +745,9 @@ def __init__(self, roots: dict[str, Path], a_root: Path, seed: Seed, a: str): self.needles = needles | {Path(n).as_posix().lower() for n in needles} self.observed: dict[str, dict[str, list[str]]] = {} - def check(self, label: str, response: Any, request: Any) -> None: + def check( + self, label: str, response: Any, request: Any, model: type[BaseModel] + ) -> None: echoed = set(_strings(request)) for value in _strings(response): for secret in self.forbidden: @@ -674,19 +755,31 @@ def check(self, label: str, response: Any, request: Any) -> None: # A bare canary may appear only as an exact echo of the request # (memory_verify returns its claim). assert CANARY not in value or value in echoed, f"{label}: {value!r}" - lowered = value.lower() - for needle in self.needles: - assert needle not in lowered, f"{label}: host path in {value!r}" + leaks = _host_path_leaks(model, response, self.needles) + assert not leaks, f"{label}: host path in {leaks!r}" assert _ledger_counts(self.a_root) == self.a_ledger, f"{label} changed A" async def run(self, client, name: str, target: str, root: Path) -> None: invoke = _invoker(client) observed = self.observed.setdefault(name, {}) assert (await invoke("session_brief", {"workspace_id": target}))["ok"] + # The target's own note names its root. Reads echo it only in + # UserText fields; the leak check must pass without a name exemption. + # The marker is in the file name too, so a file entity the graph + # extracts from the note (POSIX paths only) carries it as well. + marked = root / "src" / f"{NOTE_MARKER}.py" + note = { + "record_type": "warning", + "content": f"Target note {NOTE_MARKER}: see {marked}", + "idempotency_key": f"sweep-{name}-host-note-0001", + } + assert (await _protected(invoke, target, "memory_store", note))["ok"] own: dict[str, str] = {} for tool, arguments in OWN_VARIANTS: response = await _target_call(invoke, target, tool, arguments) - self.check(f"{name}:{tool} (own)", response, arguments) + self.check( + f"{name}:{tool} (own)", response, arguments, _response_model(tool) + ) own[tool] = _code(response) if tool == "code_search" and APPS_AVAILABLE: # The scan must cover real results, not an empty page. @@ -705,19 +798,16 @@ async def run(self, client, name: str, target: str, root: Path) -> None: for variant in variants: before = _ledger_counts(root) response = await _target_call(invoke, target, tool, variant) - self.check(f"{name}:{tool}", response, variant) + self.check(f"{name}:{tool}", response, variant, _response_model(tool)) if not response["ok"]: assert _ledger_counts(root) == before, f"{name}:{tool} wrote" observed.setdefault(tool, []).append(_code(response)) for tool, code in own.items(): observed[tool].append(code) - templates = [ - f"memory://workspaces/{target}/{kind}" - for kind in ("warnings", "failures", "rules", "active-context") - ] - for uri in templates: - contents = await client.read_resource(uri) - self.check(uri, [json.loads(item.text) for item in contents], uri) + for kind, model in RESOURCE_MODELS.items(): + uri = f"memory://workspaces/{target}/{kind}" + for item in await client.read_resource(uri): + self.check(uri, json.loads(item.text), uri, model) def assert_pinned(self, name: str) -> None: for tool, codes in sorted(self.observed[name].items()): @@ -802,7 +892,7 @@ async def _cross_workspace_checks(invoke, sweep, a, b, b_root, seed) -> list[str async def refused(tool, arguments, code): before = _ledger_counts(b_root) response = await _target_call(invoke, b, tool, arguments) - sweep.check(f"B:{tool}", response, arguments) + sweep.check(f"B:{tool}", response, arguments, _response_model(tool)) assert _code(response) == code, (tool, response) assert _ledger_counts(b_root) == before, tool diff --git a/tests/api_v7/test_production.py b/tests/api_v7/test_production.py index c6f1ac2..8543d66 100644 --- a/tests/api_v7/test_production.py +++ b/tests/api_v7/test_production.py @@ -98,7 +98,7 @@ def test_native_runtime_preloads_only_each_enabled_profile( "sentence_transformers", "onnx", "onnxruntime", - "numpy", + # numpy belongs to models-local; the graph extra never installs it. "networkx", "igraph", "leidenalg", @@ -106,6 +106,46 @@ def test_native_runtime_preloads_only_each_enabled_profile( imported, ) + def test_a_broken_installed_runtime_degrades_its_own_profile_only(self) -> None: + """Metadata can say ready while the package cannot import here.""" + from daem0nmcp.api.v7 import production + + def import_module(name: str) -> object: + if name == "igraph": + raise OSError("[WinError 126] The specified module could not be found") + return object() + + statuses = {"graph": "ready", "local": "ready"} + with patch.object(production.importlib, "import_module", import_module): + lifecycle = production._OptionalNativeRuntimeLifecycle(statuses) + lifecycle.start() + + self.assertEqual({"graph": "degraded", "local": "ready"}, statuses) + self.assertIn("igraph", lifecycle.failures["graph"]) + self.assertIn("OSError", lifecycle.failures["graph"]) + + states = { + state.name: state + for state in production._capability_states( + {"DAEM0NMCP_GRAPH_ENABLED": "true"}, lifecycle.failures + ) + } + self.assertEqual("degraded", states["graph"].status) + self.assertIn("igraph", str(states["graph"].remediation)) + + def test_capability_states_carry_the_registry_remediation(self) -> None: + """Health must say how to enable a profile, not 'review' it.""" + from daem0nmcp.api.v7 import production + + states = { + state.name: state + for state in production._capability_states( + {"DAEM0NMCP_GRAPH_ENABLED": "false"} + ) + } + self.assertEqual("disabled", states["graph"].status) + self.assertIn("DAEM0NMCP_GRAPH_ENABLED", str(states["graph"].remediation)) + def test_full_manifest_rejects_missing_or_unexpected_resources(self) -> None: from daem0nmcp.api.v7.production import build_production_surface from daem0nmcp.api.v7.registry import ManifestError @@ -409,6 +449,8 @@ async def snapshot(_workspace, **limits): ) self.assertEqual(len(calls), 1) + # Guidance never shows git changes, so it must not spawn git for them. + self.assertIs(calls[0]["include_git_changes"], False) self.assertEqual(result["records"], [relevant_failure]) self.assertEqual(result["rules"], [relevant_rule]) self.assertIn( @@ -600,5 +642,66 @@ def build_server(_surface, **options): self.assertEqual([False], observed) +async def test_memory_preflight_spawns_no_subprocess(tmp_path): + """Preflight guidance must not pay for git (F-040); session_brief still does.""" + import subprocess + + from fastmcp import Client + + from daem0nmcp.api.v7.production import create_v7_server + from daem0nmcp.retrieval.runtime import await_projection_job_drains + from daem0nmcp.storage_activation import resolve_active_database + from tests.api_v7.process_client import initialize_workspaces + + [workspace] = await initialize_workspaces((tmp_path,)) + database = resolve_active_database(tmp_path / ".daem0nmcp" / "storage").path + spawned: list[list[str]] = [] + + def record_spawn(argv, *args, **kwargs): + spawned.append(list(argv)) + raise FileNotFoundError(argv[0]) + + server = create_v7_server( + "stdio", + settings=Settings( + project_root=str(tmp_path), + workspace_roots=[str(tmp_path)], + dream_enabled=False, + ), + environ={}, + ) + scope = {"workspace_id": workspace.workspace_id} + async with Client(server) as client: + with patch.object(subprocess, "Popen", side_effect=record_spawn): + brief = await client.call_tool("session_brief", scope, raise_on_error=False) + assert brief.structured_content["ok"], brief.structured_content + assert any(argv[0] == "git" for argv in spawned), spawned + # ``Popen`` is patched process-wide, so anything running + # concurrently lands in ``spawned`` too. The brief's write + # schedules a projection drain, and with the graph profile + # installed that drain rebuilds communities on its own thread + # and spawns the Leiden worker. Let the drains it scheduled + # finish before the window opens, so the assertion below can + # stay literal -- no subprocess at all -- instead of being + # narrowed to git or excused for a thread name. + await await_projection_job_drains((database,)) + spawned.clear() + preflight = await client.call_tool( + "memory_preflight", + { + **scope, + "target_tool": "memory_store", + "target_arguments": { + "record_type": "decision", + "content": "Preflight must not spawn processes.", + "idempotency_key": "preflight-no-subprocess-0001", + }, + }, + raise_on_error=False, + ) + assert preflight.structured_content["ok"], preflight.structured_content + assert spawned == [] + + if __name__ == "__main__": unittest.main() diff --git a/tests/api_v7/test_resource_repository.py b/tests/api_v7/test_resource_repository.py index 45a43a4..5805f68 100644 --- a/tests/api_v7/test_resource_repository.py +++ b/tests/api_v7/test_resource_repository.py @@ -469,22 +469,21 @@ async def test_memory_flags_are_honored_without_losing_row_state(self) -> None: self.assertEqual(by_id[deleted].item.current_status, "invalidated") self.assertTrue(by_id[deleted].deleted) - async def test_canonical_root_embedded_in_public_text_fails_closed(self) -> None: - # Catches free-text fields bypassing the path-safe structured field policy. - self.fixture.add_record( - 12, - content=f"do not expose {self.fixture.workspace_root / 'secret.py'}", - ) + async def test_user_text_naming_the_workspace_root_reads_back(self) -> None: + # User text may mention paths, the root included (UD-3); one such + # warning must not make the resource unreadable. + content = f"see {self.fixture.workspace_root / 'app.py'}" + record_id = self.fixture.add_record(12, content=content) repository = self._repository() - from daem0nmcp.api.v7.resource_repository import ResourceRepositoryError - with self.assertRaises(ResourceRepositoryError): - await repository.read_warnings( - self.fixture.workspace, - ResourceReadRequest( - kind="warnings", limit=1, order_by="updated_at_desc" - ), - ) + rows = await repository.read_warnings( + self.fixture.workspace, + ResourceReadRequest(kind="warnings", limit=1, order_by="updated_at_desc"), + ) + + self.assertEqual( + [(record_id, content)], [(r.item.record_id, r.item.excerpt) for r in rows] + ) async def test_rules_use_canonical_ids_and_highest_priority_enabled_order( self, @@ -716,8 +715,10 @@ def resolve(_workspace: Workspace) -> ResolvedActiveDatabase: self.assertFalse(read.done()) self.assertEqual(pool.in_flight, 1) release.set() - with self.assertRaises(ResourceRepositoryError): + with self.assertRaises(ResourceRepositoryError) as caught: await read + # An overrun read that did not itself fail is retryable contention. + self.assertEqual(caught.exception.code, "DATABASE_IN_USE") self.assertEqual(pool.in_flight, 0) finally: release.set() @@ -836,6 +837,32 @@ def read_records(*args, **kwargs): ) self.assertEqual(snapshot.workspace_statistics["active_context"], 2) + async def test_snapshot_skips_git_subprocesses_when_changes_are_unused( + self, + ) -> None: + # Preflight guidance paid two git subprocesses per call for data it + # discarded; under load that pushed reads past their deadline. + from daem0nmcp.api.v7.resource_repository import SQLiteResourceRepository + + repository = SQLiteResourceRepository( + lambda _workspace: self.fixture.resolved, + clock=lambda: NOW, + ) + git_reads: list[object] = [] + + def read_git_changes(workspace): + git_reads.append(workspace) + return [] + + repository._read_git_changes_sync = read_git_changes + limits = {"warning_limit": 1, "failure_limit": 1, "active_context_limit": 1} + skipped = await repository.read_briefing_snapshot( + self.fixture.workspace, include_git_changes=False, **limits + ) + self.assertEqual((skipped.git_changes, git_reads), ([], [])) + await repository.read_briefing_snapshot(self.fixture.workspace, **limits) + self.assertEqual(len(git_reads), 1) + async def test_briefing_sections_share_one_sqlite_read_snapshot(self) -> None: # A canonical writer may commit while holding the same shared generation # lock. Every section of one brief must still observe one SQLite snapshot. diff --git a/tests/api_v7/test_runtime_services.py b/tests/api_v7/test_runtime_services.py index 7f31a5f..70ee4fb 100644 --- a/tests/api_v7/test_runtime_services.py +++ b/tests/api_v7/test_runtime_services.py @@ -125,7 +125,7 @@ async def test_safe_resolver_requires_a_format_7_active_pointer(self) -> None: (other_storage / "daem0nmcp.db").touch() pointerless = WorkspaceRegistry([other], default_root=other).default with ( - self.assertRaisesRegex(RuntimeServiceError, "ACTIVE_V7_UNAVAILABLE"), + self.assertRaisesRegex(RuntimeServiceError, "MIGRATION_REQUIRED"), resolver.locked_active(pointerless), ): self.fail("pointerless storage was admitted") @@ -332,6 +332,7 @@ async def test_outcome_updates_existing_workspace_record_and_replays( async def test_cancelled_worker_retains_capacity_and_leaves_no_write(self) -> None: """Cancelling an asyncio waiter must not over-admit or commit later.""" + from daem0nmcp.api.v7.runtime_services import RuntimeServiceError from daem0nmcp.bounded_workers import BoundedWorkerBusyError clock_entered = threading.Event() @@ -347,11 +348,13 @@ def blocked_clock() -> datetime: await asyncio.to_thread(clock_entered.wait, 1.0) first.cancel() await asyncio.sleep(0) - with self.assertRaises(BoundedWorkerBusyError): + with self.assertRaises(RuntimeServiceError) as busy: await writer.store( self.workspace, self._store_command(idempotency_key="runtime-store-overflow"), ) + self.assertEqual("DATABASE_IN_USE", busy.exception.code) + self.assertIsInstance(busy.exception.__cause__, BoundedWorkerBusyError) release_clock.set() with self.assertRaises(asyncio.CancelledError): await first @@ -830,16 +833,17 @@ async def retrieve(self, _query): ): await run(self._retrieval_result(stored, provider="")) + # A row migrated from v6 keeps the host-absolute ``file_path`` v6 + # wrote. It is legacy provenance, so it must not deny the recall, and + # it must never leave the server. with closing(sqlite3.connect(self.database)) as connection: connection.execute( "UPDATE memory_records SET file_path=? WHERE record_id=?", (str(self.root / "private.txt"), stored.record.record_id), ) connection.commit() - with self.assertRaisesRegex( - RuntimeServiceError, "EVIDENCE_AUTHENTICATION_FAILED" - ): - await run(self._retrieval_result(stored)) + legacy = await run(self._retrieval_result(stored)) + self.assertNotIn("private.txt", legacy.model_dump_json()) async def test_recall_rejects_federation_and_preserves_abstention( self, @@ -1239,7 +1243,7 @@ async def leaky_reader(_workspace, _request): with self.assertRaisesRegex( RuntimeServiceError, - "UNSAFE_SERVICE_OUTPUT", + "BRIEFING_FAILED", ) as leak: await BasicBriefingService(reader=leaky_reader).assemble( self.workspace, diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index 8459d84..ef5712c 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -218,6 +218,95 @@ def test_disabled_capability_does_not_probe_its_optional_modules(self): "DAEM0NMCP_MODELS_LOCAL_ENABLED=true", ) + def test_unset_variable_enables_an_installed_profile(self): + """An installed extra turns its profile on without any variable.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={}, module_available=lambda _: True + ).get("graph") + + self.assertEqual(capability["status"], "ready") + + def test_unset_variable_leaves_a_missing_extra_disabled(self): + """Without the extra, an unrequested profile is off, not degraded.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={}, module_available=lambda name: name != "leidenalg" + ).get("graph") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual( + capability["remediation"], + { + "action": "install_extra", + "command": "pip install 'daem0nmcp[graph]'", + "missing": ["leidenalg"], + }, + ) + + def test_an_empty_value_is_treated_as_unset(self): + """An empty variable means auto, as an unset one does.""" + from daem0nmcp.capabilities import CapabilityRegistry + + installed = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": " "}, + module_available=lambda _: True, + ).get("graph") + missing = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": ""}, + module_available=lambda _: False, + ).get("graph") + + self.assertEqual(installed["status"], "ready") + self.assertEqual(missing["status"], "disabled") + self.assertEqual(missing["remediation"]["action"], "install_extra") + + def test_explicit_false_disables_an_installed_profile(self): + """The variable still turns an installed profile off, without probing.""" + from daem0nmcp.capabilities import CapabilityRegistry + + def probe(distribution): + raise AssertionError(f"probed {distribution} for a disabled profile") + + capability = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": "false"}, + module_available=probe, + ).get("graph") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual(capability["remediation"]["action"], "enable_configuration") + self.assertIn( + "Unset DAEM0NMCP_GRAPH_ENABLED or set it to true", + capability["remediation"]["message"], + ) + + def test_explicit_true_with_a_missing_extra_is_degraded(self): + """Asking for a profile whose packages are absent is a degraded request.""" + from daem0nmcp.capabilities import CapabilityRegistry + + capability = CapabilityRegistry( + environ={"DAEM0NMCP_GRAPH_ENABLED": "true"}, + module_available=lambda name: name != "leidenalg", + ).get("graph") + + self.assertEqual(capability["status"], "degraded") + self.assertEqual(capability["remediation"]["action"], "install_extra") + self.assertEqual(capability["remediation"]["missing"], ["leidenalg"]) + + def test_unset_models_local_is_disabled_on_unsupported_python(self): + """Auto mode never reports an unrequested profile as degraded.""" + from daem0nmcp.capabilities import CapabilityRegistry + + with patch("daem0nmcp.capabilities.sys.version_info", (3, 10, 21)): + capability = CapabilityRegistry( + environ={}, module_available=lambda _: True + ).get("models-local") + + self.assertEqual(capability["status"], "disabled") + self.assertEqual(capability["remediation"]["action"], "upgrade_python") + def test_invalid_enablement_value_is_failed_with_structured_remediation(self): """An invalid profile setting is a configuration failure, not a missing extra.""" from daem0nmcp.capabilities import CapabilityRegistry @@ -234,6 +323,7 @@ def test_invalid_enablement_value_is_failed_with_structured_remediation(self): self.assertEqual( capability["remediation"]["environment"], "DAEM0NMCP_MODELS_LOCAL_ENABLED" ) + self.assertIn("unset", capability["remediation"]["message"]) def test_ready_profile_checks_only_the_requested_profile(self): """Capability inspection must not import or probe every optional subsystem.""" diff --git a/tests/test_cli_retrieval.py b/tests/test_cli_retrieval.py index 057f712..5c60a42 100644 --- a/tests/test_cli_retrieval.py +++ b/tests/test_cli_retrieval.py @@ -9,6 +9,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch from tests.test_migrate_v7 import _create_legacy_database @@ -157,6 +158,66 @@ def test_registered_secondary_workspace_opens_its_own_active_database(self): self.assertEqual(1, len(lexical)) self.assertEqual(second_id, payload["workspace_id"]) + def test_compact_projections_drains_superseded_generations_offline(self): + """Databases bloated before the GC existed need a one-shot drain.""" + + import sqlite3 + + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.storage_activation import resolve_active_database + + with tempfile.TemporaryDirectory() as raw: + root, workspace_id = self._migrated_fixture(raw) + database = resolve_active_database(root / ".daem0nmcp" / "storage").path + connection = sqlite3.connect(database) + try: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(9): + LexicalProjectionBuilder(connection).rebuild(workspace_id) + superseded = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical' AND status<>'active'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(9, superseded) + + result = self._run( + root, + "compact-projections", + "--workspace-id", + workspace_id, + ) + + self.assertEqual(0, result.returncode, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(8, payload["collected"]["lexical"]) + self.assertLess( + payload["superseded_manifests_after"], + payload["superseded_manifests_before"], + ) + self.assertTrue(payload["vacuumed"]) + self.assertNotIn(str(root), result.stdout) + connection = sqlite3.connect(database) + try: + remaining = connection.execute( + "SELECT COUNT(*) FROM projection_manifests " + "WHERE projection_name='lexical'" + ).fetchone()[0] + tables = connection.execute( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' " + "AND name GLOB 'retrieval_fts_*_g*' " + "AND sql GLOB 'CREATE VIRTUAL TABLE*'" + ).fetchone()[0] + finally: + connection.close() + self.assertEqual(2, remaining) + self.assertEqual(2, tables) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_event_store.py b/tests/test_event_store.py index 5d2144d..6e3066a 100644 --- a/tests/test_event_store.py +++ b/tests/test_event_store.py @@ -257,6 +257,103 @@ def test_expected_version_is_exactly_idempotent_or_conflicts(self): self.connection.execute("SELECT count(*) FROM memory_events").fetchone()[0], ) + def test_only_integrity_errors_on_insert_are_stream_conflicts(self): + """A lock or I/O failure must not be reported as a retryable conflict.""" + command_type, store_type = self._api() + from daem0nmcp.event_store import EventStreamConflict + + connection = self.connection + + class FailingInsert: + error: Exception + + def execute(self, sql, *arguments): + if "INSERT INTO memory_events" in sql: + raise self.error + return connection.execute(sql, *arguments) + + def __getattr__(self, name): + return getattr(connection, name) + + command = command_type( + workspace_id=self.workspace_id, + stream_id="mem_" + "c" * 64, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=1, + recorded_at_us=1, + actor_type="system", + payload={"record": self._state()}, + ) + failing = FailingInsert() + failing.error = sqlite3.OperationalError("disk I/O error") + with self.assertRaisesRegex(sqlite3.OperationalError, "disk I/O error"): + store_type(failing).append_and_project(command) + failing.error = sqlite3.IntegrityError("UNIQUE constraint failed") + with self.assertRaises(EventStreamConflict): + store_type(failing).append_and_project(command) + connection.rollback() + + def test_only_integrity_errors_on_governance_insert_are_conflicts(self): + """The governance append follows the same conflict rule as memory.""" + from daem0nmcp.event_store import ( + EventStreamConflict, + GovernanceEventCommand, + GovernanceEventStore, + ) + from daem0nmcp.migrations.schema import MIGRATIONS + from daem0nmcp.schema_version import CURRENT_SCHEMA_VERSION + + connection = sqlite3.connect(":memory:") + self.addCleanup(connection.close) + connection.execute("PRAGMA foreign_keys=ON") + for version in range(16, CURRENT_SCHEMA_VERSION + 1): + for statement in next(m for m in MIGRATIONS if m[0] == version)[2]: + connection.execute(statement) + connection.commit() + + class FailingInsert: + error: Exception + + def execute(self, sql, *arguments): + if "INSERT INTO governance_events" in sql: + raise self.error + return connection.execute(sql, *arguments) + + def __getattr__(self, name): + return getattr(connection, name) + + command = GovernanceEventCommand( + workspace_id=self.workspace_id, + stream_id="rule_" + "d" * 64, + stream_kind="rule", + event_type="rule.created", + occurred_at_us=1, + recorded_at_us=1, + actor_type="system", + expected_stream_version=1, + payload={ + "rule_id": "rule_" + "d" * 64, + "trigger": "conflict mapping", + "must_do": [], + "must_not": [], + "ask_first": [], + "warnings": [], + "priority": 1, + "enabled": True, + "created_at_us": 1, + "updated_at_us": 1, + }, + ) + failing = FailingInsert() + failing.error = sqlite3.OperationalError("disk I/O error") + with self.assertRaisesRegex(sqlite3.OperationalError, "disk I/O error"): + GovernanceEventStore(failing).append_and_project(command) + failing.error = sqlite3.IntegrityError("UNIQUE constraint failed") + with self.assertRaises(EventStreamConflict): + GovernanceEventStore(failing).append_and_project(command) + connection.rollback() + def test_projection_failure_rolls_back_event_with_savepoint(self): """No immutable event may survive a failed projection in caller transaction.""" command_type, store_type = self._api() diff --git a/tests/test_memory.py b/tests/test_memory.py index 532269b..9f4ff41 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -306,8 +306,15 @@ async def test_v7_legacy_id_keeps_opaque_canonical_origin(self, memory_manager): assert selected["record_id"].startswith("mem_") assert selected["evidence_refs"][0]["record_id"] == selected["record_id"] assert selected["origin_workspace_id"].startswith("ws_") - # Public absolute paths are never re-exposed through legacy context. - assert selected["context"] is None + # Context is user text and may mention paths (UD-3); it comes back as + # stored, while the provenance fields stay opaque and path-free. + assert selected["context"] == {"reason": r"stored beside C:\private\source.py"} + provenance = [ + selected["record_id"], + selected["origin_workspace_id"], + *map(str, selected["evidence_refs"]), + ] + assert not any("private" in value for value in provenance), provenance @pytest.mark.asyncio async def test_failed_decisions_boosted(self, memory_manager): diff --git a/tests/test_migrate_v7.py b/tests/test_migrate_v7.py index cb4969d..2fd400d 100644 --- a/tests/test_migrate_v7.py +++ b/tests/test_migrate_v7.py @@ -1320,8 +1320,9 @@ def test_reactivation_preserves_only_the_previous_active_lexical_generation(self self.assertEqual("reactivate", reactivated.action) active = sqlite3.connect(resolve_active_database(storage).path) try: + # Generation 1 is superseded twice over and is collected. self.assertEqual( - [(1, "ready"), (2, "ready"), (3, "active")], + [(2, "ready"), (3, "active")], active.execute( "SELECT generation,status FROM projection_manifests " "WHERE projection_name='lexical' ORDER BY generation" @@ -1430,7 +1431,9 @@ def test_reactivation_validates_evolved_v7_authority_not_original_v6_rows(self): candidate.commit() candidate.close() - service.rollback(root, migrated.migration_run_id) + # The v7 event written above is deliberate, so the operator + # acknowledges that rolling back hides it until reactivation. + service.rollback(root, migrated.migration_run_id, discard_v7_writes=True) reactivated = service.apply(root) self.assertEqual("reactivate", reactivated.action) @@ -1500,7 +1503,9 @@ def test_reactivation_rejects_current_memory_compatibility_divergence(self): ) candidate.commit() candidate.close() - service.rollback(root, migrated.migration_run_id) + # The v7 event written above is deliberate, so the operator + # acknowledges that rolling back hides it until reactivation. + service.rollback(root, migrated.migration_run_id, discard_v7_writes=True) candidate = sqlite3.connect(candidate_path) candidate.execute( @@ -1621,7 +1626,9 @@ def test_reactivation_rejects_ambiguous_live_compatibility_claims(self): ) candidate.commit() candidate.close() - service.rollback(root, migrated.migration_run_id) + # The v7 event written above is deliberate, so the operator + # acknowledges that rolling back hides it until reactivation. + service.rollback(root, migrated.migration_run_id, discard_v7_writes=True) with self.assertRaisesRegex(MigrationV7Error, "VALIDATION_FAILED"): service.apply(root) diff --git a/tests/test_migrate_v7_end_to_end.py b/tests/test_migrate_v7_end_to_end.py new file mode 100644 index 0000000..7fc2aed --- /dev/null +++ b/tests/test_migrate_v7_end_to_end.py @@ -0,0 +1,1159 @@ +"""A migrated v6 workspace must be usable: brief, recall, outcome, export. + +The fixture is built the way v6 built its own database -- the retained v6 ORM +tables plus the real migration ledger capped at ``_LAST_V6_SCHEMA_VERSION`` -- +and seeded with rows shaped exactly as v6 ``remember`` wrote them, including +the host-absolute ``file_path``, the ``../outside`` relative form v6 writes for +a file above the project, unbounded tags and empty content. + +Its DDL is byte-identical to a database produced by running the v6 code at +commit 00809c6: the same twenty-three tables, and the same +``V6_SCHEMA_SHA256`` over their ``sqlite_master.sql``, which +``test_v6_fixture_uses_the_real_migration_ledger`` enforces so a v7-side edit +to a retained v6 model cannot quietly change what "a v6 database" means here. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +import sys +from pathlib import Path +from typing import Any + +import pytest +from fastmcp import Client +from sqlalchemy import create_engine + +from daem0nmcp import models +from daem0nmcp.api.v7.production import create_v7_server +from daem0nmcp.config import Settings +from daem0nmcp.migrations.schema import ( + _LAST_V6_SCHEMA_VERSION, + MIGRATIONS, + run_migrations, +) +from daem0nmcp.migrations.v7 import MigrationV7Error, MigrationV7Service +from daem0nmcp.storage_activation import resolve_active_database +from daem0nmcp.workspace import ( + WorkspaceRegistry, + is_workspace_relative_path, + relative_to_root, +) +from tests.api_v7.process_client import call, process_client + +# Every table v6's own ``init_db`` created, so the migration's source +# inventory, its logical hash and the rollback byte-identity check run over +# the table set a real v6 store has. The first seven carry rows. +V6_TABLES = ( + "memories", + "memory_versions", + "facts", + "memory_relationships", + "rules", + "context_triggers", + "active_context", + "code_entities", + "enforcement_bypass_log", + "extracted_entities", + "file_hashes", + "memory_code_refs", + "memory_communities", + "memory_entity_refs", + "project_links", + "session_state", +) +# The v6 column layout, pinned so a v7-side edit to a retained model cannot +# quietly change what "a v6 database" means here. Update it only when the v6 +# format genuinely changes, and say so in the commit. +V6_SCHEMA_SHA256 = "07c197f506e5ba368e3ee71ef132f7bd636872fc9d89432262a20292e46b8b82" +PROFILE_ENVIRONMENT = {"DAEM0NMCP_PROFILE": "core"} +TIME = "2026-01-01 00:00:00" + + +def _v6_file_paths(file_path: Path, project_path: Path) -> tuple[str, str]: + """Reproduce v6 ``_normalize_file_path`` (v6 memory.py:63-106) exactly. + + Including its inner fallback: across Windows drives ``os.path.relpath`` + raises too, and v6 then stored the resolved *absolute* path in the + ``file_path_relative`` column. + """ + + resolved = file_path.resolve() + absolute = str(resolved) + root = project_path.resolve() + try: + relative = resolved.relative_to(root).as_posix() + except ValueError: + try: + relative = os.path.relpath(resolved, start=root).replace("\\", "/") + except ValueError: + relative = resolved.as_posix() + if sys.platform == "win32": + absolute = absolute.lower() + relative = relative.lower() + return absolute, relative + + +def _v6_database(root: Path, *, project_path: Path | None = None) -> Path: + """Build and seed a format-6 store below ``root`` and return its storage.""" + + project = project_path or root + storage = root / ".daem0nmcp" / "storage" + storage.mkdir(parents=True) + database = storage / "daem0nmcp.db" + engine = create_engine(f"sqlite:///{database}") + models.Base.metadata.create_all( + engine, tables=[models.Base.metadata.tables[name] for name in V6_TABLES] + ) + engine.dispose() + run_migrations(str(database), maximum_version=_LAST_V6_SCHEMA_VERSION) + + inside_absolute, inside_relative = _v6_file_paths( + project / "src" / "db.py", project + ) + route_absolute, route_relative = _v6_file_paths(project / "src" / "app.py", project) + outside_absolute, outside_relative = _v6_file_paths( + project.parent / "other" / "helper.py", project + ) + memories = ( + ( + 1, + "decision", + "Use postgres for the analytics warehouse", + "SQLite could not keep up with the nightly rollups", + '{"ticket": "ANA-12"}', + '["database", "postgres"]', + inside_absolute, + inside_relative, + 0, + 0, + ), + ( + 2, + "warning", + "Keep the /health endpoint unauthenticated; crash was in C:\\proj\\app.py", + "Load balancer probes it", + "{}", + '["http"]', + route_absolute, + route_relative, + 1, + 0, + ), + ( + 3, + "pattern", + "The shared helper lives outside the project", + None, + "{}", + '["shared"]', + outside_absolute, + outside_relative, + 1, + 0, + ), + ( + 4, + "learning", + "Migrated rows must stay readable", + None, + "{}", + json.dumps( + ["x" * 120, "dup", "dup", "ctrl\x01char", *[f"t{n}" for n in range(40)]] + ), + None, + None, + 0, + 1, + ), + (5, "learning", "", None, "{}", "[]", None, None, 0, 0), + ) + connection = sqlite3.connect(database) + try: + for row in memories: + connection.execute( + "INSERT INTO memories (id,category,content,rationale,context,tags," + "file_path,file_path_relative,keywords,is_permanent,vector_embedding," + "outcome,worked,pinned,archived,recall_count,surprise_score," + "importance_score,source_client,source_model,created_at,updated_at) " + "VALUES (?,?,?,?,?,?,?,?,'kw',?,NULL,NULL,NULL,?,0,0,NULL,NULL," + "'claude-code','sonnet',?,?)", + (*row, TIME, TIME), + ) + connection.execute( + "UPDATE memories SET outcome='Cut the rollup to 4 minutes',worked=1 " + "WHERE id=1" + ) + connection.execute( + "INSERT INTO memory_versions (id,memory_id,version_number,content," + "rationale,context,tags,outcome,worked,change_type,change_description," + "changed_at,valid_from,valid_to,invalidated_by_version_id) " + "VALUES (1,1,1,'Use postgres',NULL,'{}','[\"database\"]',NULL,NULL," + "'created','initial',?,?,NULL,NULL)", + (TIME, TIME), + ) + connection.execute( + "INSERT INTO facts (id,content_hash,content,category,source_memory_id," + "verification_count,is_verified,tags,created_at,verified_at) " + "VALUES (1,'legacy-hash','Postgres is remote','database',1,2,1," + "'[\"db\"]',?,NULL)", + (TIME,), + ) + connection.execute( + "INSERT INTO memory_relationships (id,source_id,target_id,relationship," + "description,confidence,created_at) VALUES " + "(1,1,2,'related_to','both touch the API tier',1.0,?)", + (TIME,), + ) + connection.execute( + "INSERT INTO rules (id,trigger,trigger_keywords,must_do,must_not," + "ask_first,warnings,priority,enabled,created_at) VALUES " + "(1,'before editing the schema','schema','[\"write a migration\"]'," + "'[\"drop tables\"]','[]','[]',5,1,?)", + (TIME,), + ) + connection.execute( + "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," + "recall_topic,recall_categories,is_active,priority,created_at," + "trigger_count,last_triggered) VALUES " + "(1,?,'file_pattern','src/*.py','source conventions'," + "'[\"decision\"]',1,3,?,0,NULL)", + (str(project), TIME), + ) + connection.execute( + "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," + "recall_topic,recall_categories,is_active,priority,created_at," + "trigger_count,last_triggered) VALUES " + "(2,?,'entity_match','Postgres.*','storage entities'," + "'[\"decision\"]',1,1,?,0,NULL)", + (str(project), TIME), + ) + connection.execute( + "INSERT INTO active_context (id,project_path,memory_id,priority,reason," + "added_at,expires_at) VALUES (1,?,1,4,'migration focus',?,NULL)", + (str(project), TIME), + ) + connection.commit() + finally: + connection.close() + return storage + + +def _v6_schema_digest(database: Path) -> str: + """A digest of the v6 tables' DDL, in a stable order.""" + + connection = sqlite3.connect(database) + try: + statements = [ + str(row[0]) + for row in connection.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL " + "ORDER BY name" + ) + ] + finally: + connection.close() + return hashlib.sha256("\n".join(statements).encode("utf-8")).hexdigest() + + +def _registry(root: Path) -> WorkspaceRegistry: + return WorkspaceRegistry([root], default_root=root) + + +def _workspace_id(root: Path) -> str: + return _registry(root).default.workspace_id + + +def _server(root: Path): + return create_v7_server( + "stdio", + settings=Settings( + project_root=str(root), + workspace_roots=[str(root)], + dream_enabled=False, + ), + environ=PROFILE_ENVIRONMENT, + ) + + +def _records(storage: Path) -> list[sqlite3.Row]: + connection = sqlite3.connect(resolve_active_database(storage).path) + connection.row_factory = sqlite3.Row + try: + return connection.execute( + "SELECT * FROM memory_records ORDER BY created_at_us,record_id" + ).fetchall() + finally: + connection.close() + + +async def _invoke(client: Client, workspace_id: str, tool: str, **arguments: Any): + result = await client.call_tool( + tool, {"workspace_id": workspace_id, **arguments}, raise_on_error=False + ) + if result.structured_content is None: + # A schema rejection by FastMCP never reaches the v7 router. + assert result.is_error, tool + return {"ok": False, "error": {"code": "SCHEMA_REJECTED"}} + return result.structured_content + + +async def _succeed(client: Client, workspace_id: str, tool: str, **arguments: Any): + response = await _invoke(client, workspace_id, tool, **arguments) + assert response["ok"], (tool, response["error"]) + return response["data"] + + +async def _protected( + client: Client, workspace_id: str, tool: str, arguments: dict[str, Any] +): + """Call a covenant-protected tool with the preflight token it requires.""" + + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool=tool, + target_arguments=arguments, + description=f"review before {tool}", + ) + return await _succeed( + client, + workspace_id, + tool, + preflight_token=preflight["preflight_token"], + **arguments, + ) + + +def test_v6_fixture_uses_the_real_migration_ledger(tmp_path): + """The fixture is a chain-built v6 store, not a hand-written table set.""" + + storage = _v6_database(tmp_path / "project") + connection = sqlite3.connect(storage / "daem0nmcp.db") + try: + versions = [ + int(row[0]) + for row in connection.execute("SELECT version FROM schema_version") + ] + finally: + connection.close() + assert max(versions) == _LAST_V6_SCHEMA_VERSION + assert set(versions) >= { + version for version, _, _ in MIGRATIONS if version <= _LAST_V6_SCHEMA_VERSION + } + # R-25: the layout itself, so a v7-side edit to a retained v6 model cannot + # change what this fixture means without failing here. + assert _v6_schema_digest(storage / "daem0nmcp.db") == V6_SCHEMA_SHA256 + + +async def test_migrated_v6_workspace_is_usable_end_to_end(tmp_path): + """F-003/F-004/F-007/F-008/F-053: migrate, brief, recall, outcome, export.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + + result = MigrationV7Service(_registry(root)).apply(root) + assert result.status == "activated", result + assert result.validation["rule_count"] == 1 + # Both v6 trigger types the fixture seeds, including ``entity_match``. + assert result.validation["trigger_count"] == 2 + # The one memory whose file lives above the project cannot keep a + # workspace-relative link, and the migration says so instead of writing a + # path that later fails every read. + assert result.validation["dropped_file_link_count"] == 1 + assert any("file link" in warning for warning in result.warnings) + + rows = {row["content"]: row for row in _records(storage)} + assert all(row["file_path"] is None for row in rows.values()) + assert rows["Use postgres for the analytics warehouse"]["file_path_relative"] == ( + "src/db.py" + ) + # The helper lives above the project: no workspace-relative link exists. + assert ( + rows["The shared helper lives outside the project"]["file_path_relative"] + is None + ) + # v6 tags had no length, uniqueness or count limit; the v7 wire has all three. + tags = json.loads(rows["Migrated rows must stay readable"]["tags_json"]) + assert len(tags) == 32 and len(set(tags)) == 32 + assert all(len(tag) <= 80 for tag in tags) + # PR 5 removed the control-character refusal; a migrated tag keeps its. + assert "ctrl\x01char" in tags + assert "" in rows, sorted(rows) + + workspace_id = _workspace_id(root) + async with process_client(root, "stdio") as session: + brief = await call(session, "session_brief", {"workspace_id": workspace_id}) + assert brief["ok"], brief["error"] + assert brief["data"]["applicable_rules"], "the migrated v6 rule must brief" + + recall = await call( + session, + "memory_recall", + {"workspace_id": workspace_id, "query": "postgres"}, + ) + assert recall["ok"], recall["error"] + linked = [ + item["record"] + for item in recall["data"]["items"] + if item["record"]["relative_file_path"] == "src/db.py" + ] + assert linked, recall["data"]["items"] + + outcome = await call( + session, + "memory_record_outcome", + { + "workspace_id": workspace_id, + "record_id": linked[0]["record_id"], + "outcome_text": "still true after the migration", + "worked": True, + "idempotency_key": "migrated-outcome-0001", + }, + ) + assert outcome["ok"], outcome["error"] + + pages: list[dict[str, Any]] = [] + arguments: dict[str, Any] = {} + while len(pages) <= 32: + page = await call( + session, + "workspace_export", + {"workspace_id": workspace_id, **arguments}, + ) + assert page["ok"], page["error"] + pages.append(page["data"]) + if page["data"]["complete"] or page["data"]["next_cursor"] is None: + break + arguments = { + "export_session_id": page["data"]["export_session_id"], + "cursor": page["data"]["next_cursor"], + "page_index": page["data"]["page_index"] + 1, + } + assert pages and pages[-1]["complete"] + # Exporting at all proves no host path reached a structured path field + # (the guard refuses the whole page); assert the surviving links too. + exported = [row for page in pages for row in page["legacy_rows"]] + assert exported + for row in exported: + relative = row["record"]["file_path_relative"] + assert relative is None or is_workspace_relative_path(relative), relative + assert "file_path" not in row["record"] + # And the event pages, where the migrated v6 row rides as `[name, + # value]` pairs -- the place the host path used to reach the wire. + assert any(page["page_kind"] == "events" for page in pages) + for page in pages: + assert str(root) not in json.dumps(page) + + +async def test_every_migrated_row_reads_back_through_the_server(tmp_path): + """A migrated row cannot deny a read, whatever v6 put in it.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + for query in ("postgres", "health", "readable", "helper"): + await _succeed(client, workspace_id, "memory_recall", query=query) + await _succeed(client, workspace_id, "memory_search_text", query=query) + for kind in ("warnings", "failures", "rules", "active-context"): + await client.read_resource(f"memory://workspaces/{workspace_id}/{kind}") + + +@pytest.mark.parametrize( + "stored_relative", + [ + # v6 wrote this for a file above the project root... + "../other/helper.py", + # ...and this when `os.path.relpath` raised across Windows drives. + "c:/other/helper.py", + # A POSIX absolute path reaches the same column the same way. + "/var/other/helper.py", + ], +) +async def test_a_database_migrated_by_the_pre_fix_code_still_reads( + tmp_path, stored_relative +): + """PR 5 carry-over: the in-place repair, on values this test controls. + + Neither the ``file_path_relative`` normalization nor the tag/content + clipping is reachable through a store this branch migrated -- the + migration already clips and nulls -- so both are exercised here by + writing the pre-fix projection back and reading it through the server. + """ + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + absolute, _relative = _v6_file_paths(tmp_path / "other" / "helper.py", root) + overlong = "y" * 120 + tags = json.dumps([overlong, "same", "same", *[f"u{n}" for n in range(40)]]) + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + # One warning row carrying every shape at once, so it is read back + # through `session_brief` and the warnings resource -- the reader + # whose failure PR 5 escalated, and one that selects by record type + # rather than through the lexical index, so no re-indexing is + # involved and the assertion cannot race a background rebuild. + connection.execute( + "UPDATE memory_records SET file_path=?,file_path_relative=?,tags_json=?," + "content='' WHERE record_type='warning'", + (absolute, stored_relative, tags), + ) + connection.commit() + record_id = str( + connection.execute( + "SELECT record_id FROM memory_records WHERE record_type='warning'" + ).fetchone()[0] + ) + finally: + connection.close() + + async with Client(_server(root)) as client: + brief = await _succeed(client, workspace_id, "session_brief") + summaries = {item["record_id"]: item for item in brief["warnings"]} + assert record_id in summaries, brief["warnings"] + record = summaries[record_id] + # The stored value is not workspace-relative, so no link is emitted -- + # and above all the host path never reaches the wire. + assert record["relative_file_path"] is None + assert absolute not in json.dumps(brief) + assert stored_relative not in json.dumps(brief) + # v6 accepted empty content; the wire needs at least one character. + assert record["excerpt"] == "" + # v6 had no tag length, uniqueness or count limit; the wire has all three. + assert len(record["tags"]) == 32 + assert len(set(record["tags"])) == 32 + assert all(len(tag) <= 80 for tag in record["tags"]) + assert overlong[:80] in record["tags"] + + for kind in ("warnings", "failures", "rules", "active-context"): + document = await client.read_resource( + f"memory://workspaces/{workspace_id}/{kind}" + ) + assert absolute not in document[0].text + + +async def test_a_pre_fix_migrated_record_still_accepts_an_outcome(tmp_path): + """R-6: the repair moves `content_hash`, and that is a decision.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + database = resolve_active_database(storage).path + connection = sqlite3.connect(database) + try: + connection.execute( + "UPDATE memory_records SET file_path_relative='../other/helper.py' " + "WHERE record_type='decision'" + ) + connection.commit() + before = connection.execute( + "SELECT record_id,content_hash FROM memory_records " + "WHERE record_type='decision'" + ).fetchone() + finally: + connection.close() + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + outcome = await _succeed( + client, + workspace_id, + "memory_record_outcome", + record_id=before[0], + outcome_text="still true after the repair", + worked=True, + idempotency_key="pre-fix-outcome-0001", + ) + assert outcome["record_id"] == before[0] + assert outcome["worked"] is True + + connection = sqlite3.connect(database) + try: + after = connection.execute( + "SELECT content_hash,file_path_relative FROM memory_records " + "WHERE record_id=?", + (before[0],), + ).fetchone() + finally: + connection.close() + # Dropping the escaping link is part of the record state, so the next + # event's content hash moves although no content changed. Pinned here so + # a client keying on `content_hash` is a known consequence, not a surprise. + assert after[1] is None + assert after[0] != before[1] + + +async def test_verify_v7_stays_valid_across_the_first_write(tmp_path): + """Plan item 8: a just-activated migration verifies, and keeps verifying. + + The migration activates manifests for tables it fills directly and + nothing rebuilds them, so before the exemption the first write moved + every migrated store to `invalid` for good. + """ + + from daem0nmcp.verification_v7 import verify_v7 + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + # Not vacuous: the migration really did record table-snapshot manifests. + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + activated = { + str(row[0]) + for row in connection.execute( + "SELECT projection_name FROM projection_manifests " + "WHERE workspace_id=? AND status='active'", + (workspace_id,), + ) + } + finally: + connection.close() + assert { + "memory_records", + "memory_fact_versions", + "memory_relationship_versions", + } <= activated + + assert verify_v7(storage, workspace_id)["status"] == "verified" + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool="memory_store", + target_arguments={ + "record_type": "decision", + "content": "written after the migration", + "idempotency_key": "verify-after-write-0001", + }, + description="write after migrating", + ) + await _succeed( + client, + workspace_id, + "memory_store", + record_type="decision", + content="written after the migration", + idempotency_key="verify-after-write-0001", + preflight_token=preflight["preflight_token"], + ) + + assert verify_v7(storage, workspace_id)["status"] == "verified" + + +async def test_reactivating_keeps_governance_authored_through_v7(tmp_path): + """Reconcile must not read "not in v6" as "deleted in v6". + + `rule_create` and `context_trigger_create` also write the retained v6 + row, so a stream with no v6 row only appears once that compatibility + write is gone -- this test removes it to pin the guard that makes + reconcile speak only for what the migration itself created. + """ + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + service = MigrationV7Service(_registry(root)) + service.apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + rule = await _protected( + client, + workspace_id, + "rule_create", + { + "trigger": "before touching the cache", + "must_do": ["flush it"], + "idempotency_key": "v7-rule-0001", + }, + ) + trigger = await _protected( + client, + workspace_id, + "context_trigger_create", + { + "trigger_type": "tag", + "pattern": "cache.*", + "recall_query": "cache guidance", + "idempotency_key": "v7-trigger-0001", + }, + ) + rule_id = rule["rule_id"] + trigger_id = trigger["trigger_id"] + + # Drop the retained v6 rows the compatibility write made, leaving two + # streams that exist only in v7 -- the shape reconcile must not touch. + # That shape is not reachable through the tools today, because + # `rule_create`/`context_trigger_create` write the retained row and + # `rule_update` writes it back; it becomes reachable once PR 10 removes + # that compatibility write, which is what this guards. + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + connection.execute("DELETE FROM rules WHERE id>1") + connection.execute("DELETE FROM context_triggers WHERE id>2") + connection.commit() + finally: + connection.close() + + # The documented recovery path: roll back accepting the stranded writes, + # then re-apply, which reactivates the retained candidate. + service.rollback(root, discard_v7_writes=True) + result = service.apply(root) + assert result.action == "reactivate", result + assert result.validation["rules_disabled"] == 0, result.validation + assert result.validation["triggers_deleted"] == 0, result.validation + + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + rule_row = connection.execute( + "SELECT enabled FROM governance_rules WHERE rule_id=?", (rule_id,) + ).fetchone() + trigger_row = connection.execute( + "SELECT deleted_at_us FROM governance_context_triggers WHERE trigger_id=?", + (trigger_id,), + ).fetchone() + migrated_rule = connection.execute( + "SELECT enabled FROM governance_rules WHERE rule_id<>?", (rule_id,) + ).fetchone() + finally: + connection.close() + assert rule_row is not None and rule_row[0] == 1, "a v7 rule was disabled" + assert trigger_row is not None and trigger_row[0] is None, ( + "a v7 trigger was deleted" + ) + assert migrated_rule is not None and migrated_rule[0] == 1 + + +async def test_a_moved_project_keeps_its_file_links(tmp_path): + """F-022: v6 recorded one project root; file links resolve against it.""" + + old_root = tmp_path / "old-name" + (old_root / "src").mkdir(parents=True) + new_root = tmp_path / "new-name" + (tmp_path / "other").mkdir() + storage = _v6_database(old_root, project_path=old_root) + # v6 kept the database inside the project, so moving the directory moves + # the store with it while every recorded path still names the old root. + shutil.move(str(old_root), str(new_root)) + storage = new_root / ".daem0nmcp" / "storage" + + result = MigrationV7Service(_registry(new_root)).apply(new_root) + assert result.status == "activated", result + # The single recorded v6 project path is this workspace's own history. + assert result.validation["trigger_count"] == 2 + assert result.validation["foreign_trigger_count"] == 0 + rows = {row["record_type"]: row for row in _records(storage)} + assert rows["decision"]["file_path_relative"] == "src/db.py" + + workspace_id = _workspace_id(new_root) + async with Client(_server(new_root)) as client: + await _succeed(client, workspace_id, "session_brief") + await _succeed(client, workspace_id, "memory_recall", query="postgres") + + +def test_the_bounded_summary_clips_what_v6_had_no_limit_for(): + """The model is the one place every stored-record reader passes through.""" + + from daem0nmcp.api.v7.models import RecordSummary + + base = { + "record_id": "mem_" + "a" * 64, + "record_type": "decision", + "tags": ["x" * 120, "dup", "dup", *[f"t{n}" for n in range(40)]], + "relative_file_path": None, + "current_status": "current", + "content_hash": "b" * 64, + "created_at": "2026-01-02T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + } + summary = RecordSummary.model_validate({**base, "excerpt": ""}) + # v6 accepted empty content; the wire needs at least one character. + assert summary.excerpt == "" + # v6 tags had no length, uniqueness or count limit. + assert len(summary.tags) == 32 + assert len(set(summary.tags)) == 32 + assert all(len(tag) <= 80 for tag in summary.tags) + # A valid time ahead of the transaction time is shown as the earlier one, + # not refused: v6 kept the two independently. + assert summary.created_at == summary.updated_at + + +def test_a_sibling_directory_is_not_inside_the_workspace(tmp_path): + """R-25: containment is decided by path parts, never a string prefix.""" + + inside = tmp_path / "proj" + sibling = tmp_path / "proj2" + (inside / "src").mkdir(parents=True) + sibling.mkdir() + (inside / "src" / "a.py").write_text("", encoding="utf-8") + (sibling / "a.py").write_text("", encoding="utf-8") + + assert relative_to_root(inside / "src" / "a.py", inside) == "src/a.py" + assert relative_to_root(sibling / "a.py", inside) is None + assert relative_to_root(inside, inside) is None + assert relative_to_root(tmp_path / "proj" / ".." / "proj2" / "a.py", inside) is None + + +async def test_rollback_refuses_to_hide_writes_made_since_activation(tmp_path): + """F-024/F-068: rollback is loud about the v7 events it would strand.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + source_bytes = (storage / "daem0nmcp.db").read_bytes() + service = MigrationV7Service(_registry(root)) + service.apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool="memory_store", + target_arguments={ + "record_type": "decision", + "content": "written after the migration", + "idempotency_key": "post-migration-0001", + }, + description="write after migrating", + ) + stored = await _succeed( + client, + workspace_id, + "memory_store", + record_type="decision", + content="written after the migration", + idempotency_key="post-migration-0001", + preflight_token=preflight["preflight_token"], + ) + assert stored["record"]["record_id"] + + with pytest.raises(MigrationV7Error) as refusal: + service.rollback(root) + assert refusal.value.code == "ROLLBACK_WOULD_HIDE_WRITES" + # The operator is told how much would be lost and where it stays. + assert "1 event(s)" in str(refusal.value) + assert "candidate.db" in str(refusal.value) + assert "--discard-v7-writes" in str(refusal.value) + assert resolve_active_database(storage).format_version == 7 + + result = service.rollback(root, discard_v7_writes=True) + assert result.status == "rolled_back" + assert any( + "1 event(s)" in warning and "candidate.db" in warning + for warning in result.warnings + ), result.warnings + assert resolve_active_database(storage).format_version == 6 + assert (storage / "daem0nmcp.db").read_bytes() == source_bytes + + +@pytest.mark.parametrize( + "table,insert", + [ + ( + "workspace_link_events", + "INSERT INTO workspace_link_events (event_id,workspace_id," + "linked_workspace_id,stream_version,event_type,occurred_at_us," + "recorded_at_us,event_hash) VALUES (?,?,?,1,'workspace.linked'," + "?,?,?)", + ), + ( + "memory_capture_candidates", + "INSERT INTO memory_capture_candidates (candidate_id,workspace_id," + "idempotency_key,candidate_hash,source_kind,proposed_record_json," + "provenance_json,created_at_us) VALUES (?,?,'capture-key-0001',?," + "'native_edit','{}','{}',?)", + ), + ], +) +def test_rollback_sees_every_stream_a_client_can_write(tmp_path, table, insert): + """R-2/R-13/R-34: federation events and capture candidates count too.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + service = MigrationV7Service(_registry(root)) + service.apply(root) + workspace_id = _workspace_id(root) + + digest = "a" * 64 + linked = "ws_" + "b" * 24 + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + if table == "workspace_link_events": + connection.execute( + insert, ("evt_" + digest, workspace_id, linked, 1, 1, digest) + ) + else: + connection.execute(insert, ("cap_" + digest, workspace_id, digest, 1)) + connection.commit() + finally: + connection.close() + + with pytest.raises(MigrationV7Error) as refusal: + service.rollback(root) + assert refusal.value.code == "ROLLBACK_WOULD_HIDE_WRITES" + + +async def test_an_un_migrated_v6_workspace_says_migration_is_required(tmp_path): + """F-066/F-069/F-043: a non-retryable error naming the migrate command.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + _v6_database(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + for tool in ("session_brief", "memory_recall", "memory_store"): + arguments: dict[str, Any] = {} + if tool == "memory_recall": + arguments = {"query": "anything"} + elif tool == "memory_store": + arguments = { + "record_type": "decision", + "content": "anything", + "idempotency_key": "v6-store-0001", + } + response = await _invoke(client, workspace_id, tool, **arguments) + assert response["ok"] is False, tool + # Store and recall are gated behind a briefing that cannot + # succeed on a v6 store, so they answer COMMUNION_REQUIRED first. + assert response["error"]["code"] in { + "MIGRATION_REQUIRED", + "COMMUNION_REQUIRED", + "SCHEMA_REJECTED", + }, (tool, response["error"]) + brief = await _invoke(client, workspace_id, "session_brief") + assert brief["error"]["code"] == "MIGRATION_REQUIRED" + assert brief["error"]["retryable"] is False + assert "migrate-v7 --apply" in brief["error"]["message"] + # The resource surface deliberately answers one invariant + # (`RESOURCE_UNAVAILABLE`) for every failure, so a caller cannot + # enumerate workspaces by reading its errors; `session_brief` is + # where the reason is named. + from mcp.shared.exceptions import McpError + + with pytest.raises(McpError): + await client.read_resource(f"memory://workspaces/{workspace_id}/rules") + + # The transfer boundary raises it too. It sits behind the covenant, so + # reach it the way an operator would: brief a migrated store, roll it back + # to v6 under the live session, then export. + migrated = tmp_path / "migrated" + (migrated / "src").mkdir(parents=True) + migrated_storage = _v6_database(migrated) + service = MigrationV7Service(_registry(migrated)) + service.apply(migrated) + migrated_id = _workspace_id(migrated) + async with Client(_server(migrated)) as client: + await _succeed(client, migrated_id, "session_brief") + service.rollback(migrated, discard_v7_writes=True) + assert resolve_active_database(migrated_storage).format_version == 6 + exported = await _invoke(client, migrated_id, "workspace_export") + assert exported["ok"] is False + assert exported["error"]["code"] == "MIGRATION_REQUIRED", exported["error"] + + +def test_a_v6_rule_edited_after_an_in_place_upgrade_reaches_v7(tmp_path): + """F-023: a `must_not` added after an in-place upgrade must not vanish.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + database = storage / "daem0nmcp.db" + # The pre-fix legacy CLI applied the whole ledger to the v6 file, which + # backfilled governance from the rules table as it stood then. + run_migrations(str(database), workspace_id=_workspace_id(root)) + connection = sqlite3.connect(database) + try: + connection.execute( + "UPDATE rules SET must_not=? WHERE id=1", + (json.dumps(["drop tables", "truncate"]),), + ) + connection.execute("DELETE FROM context_triggers WHERE id=1") + connection.commit() + finally: + connection.close() + + result = MigrationV7Service(_registry(root)).apply(root) + assert result.status == "activated", result + assert result.validation["rules_updated"] == 1 + assert result.validation["triggers_deleted"] == 1 + + connection = sqlite3.connect(resolve_active_database(storage).path) + try: + must_not = connection.execute( + "SELECT must_not_json FROM governance_rules" + ).fetchone()[0] + deleted = connection.execute( + "SELECT count(*) FROM governance_context_triggers " + "WHERE deleted_at_us IS NOT NULL" + ).fetchone()[0] + finally: + connection.close() + assert json.loads(must_not) == ["drop tables", "truncate"] + assert deleted == 1 + + +def test_a_foreign_project_trigger_is_skipped_and_counted(tmp_path): + """F-022: triggers from other project roots are reported, not dropped.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + connection = sqlite3.connect(storage / "daem0nmcp.db") + try: + connection.execute( + "INSERT INTO context_triggers (id,project_path,trigger_type,pattern," + "recall_topic,recall_categories,is_active,priority,created_at," + "trigger_count,last_triggered) VALUES " + "(3,?,'tag_match','auth.*','authentication','[\"warning\"]',1,1,?,0,NULL)", + (str(tmp_path / "another-project"), TIME), + ) + connection.commit() + finally: + connection.close() + + result = MigrationV7Service(_registry(root)).apply(root) + assert result.validation["trigger_count"] == 2 + assert result.validation["foreign_trigger_count"] == 1 + assert any("another project path" in warning for warning in result.warnings) + + +async def test_export_pages_come_from_one_snapshot(tmp_path): + """Writes between pages cannot change a page already in the session.""" + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + _v6_database(root) + MigrationV7Service(_registry(root)).apply(root) + workspace_id = _workspace_id(root) + + async with Client(_server(root)) as client: + await _succeed(client, workspace_id, "session_brief") + first = await _succeed( + client, workspace_id, "workspace_export", page_byte_limit=65_536 + ) + assert first["page_count"] >= 2, "the fixture must page" + preflight = await _succeed( + client, + workspace_id, + "memory_preflight", + target_tool="memory_store", + target_arguments={ + "record_type": "decision", + "content": "written between export pages", + "idempotency_key": "between-pages-0001", + }, + description="write between export pages", + ) + await _succeed( + client, + workspace_id, + "memory_store", + record_type="decision", + content="written between export pages", + idempotency_key="between-pages-0001", + preflight_token=preflight["preflight_token"], + ) + page = first + collected = [first] + while not page["complete"] and page["next_cursor"] is not None: + page = await _succeed( + client, + workspace_id, + "workspace_export", + export_session_id=page["export_session_id"], + cursor=page["next_cursor"], + page_index=page["page_index"] + 1, + ) + collected.append(page) + assert page["complete"] + # Every page belongs to the session's manifest, taken before the write. + assert {item["manifest_hash"] for item in collected} == {first["manifest_hash"]} + assert all(item["root_hash"] == first["root_hash"] for item in collected), ( + "a later page must not observe an event written after the session" + ) + + +def test_cli_status_leaves_a_v6_database_at_its_last_v6_version(tmp_path): + """F-023/A-005: a legacy command must not upgrade the migration source.""" + + import subprocess + + root = tmp_path / "project" + (root / "src").mkdir(parents=True) + (tmp_path / "other").mkdir() + storage = _v6_database(root) + database = storage / "daem0nmcp.db" + + completed = subprocess.run( + [ + sys.executable, + "-m", + "daem0nmcp.cli", + "--project-path", + str(root), + "--json", + "status", + ], + capture_output=True, + text=True, + timeout=300, + ) + assert completed.returncode == 0, completed.stderr[-2000:] + connection = sqlite3.connect(database) + try: + version = connection.execute( + "SELECT MAX(version) FROM schema_version" + ).fetchone()[0] + finally: + connection.close() + assert version == _LAST_V6_SCHEMA_VERSION + assert resolve_active_database(storage).format_version == 6 + # The source is still migratable afterwards, and the store the ORM's own + # DDL helped build still satisfies the v7 physical-schema contract. + from daem0nmcp.verification_v7 import verify_v7 + + assert MigrationV7Service(_registry(root)).apply(root).status == "activated" + assert verify_v7(storage, _workspace_id(root))["status"] == "verified" diff --git a/tests/test_opencode_install.py b/tests/test_opencode_install.py index a94b63a..8b0c0c6 100644 --- a/tests/test_opencode_install.py +++ b/tests/test_opencode_install.py @@ -11,6 +11,7 @@ from daem0nmcp.edit_bridge_transport import provision_bridge_credential from daem0nmcp.opencode_install import ( + COMMAND_TEMPLATES, PLUGIN_TEMPLATE, install_opencode, select_opencode_interface, @@ -228,3 +229,80 @@ def test_install_preserves_existing_opencode_json(tmp_path): assert plugin.exists(), ( "Plugin file should be created even when opencode.json exists" ) + + +def test_install_creates_repo_command_files_without_overwriting(tmp_path): + """The installer ships the repo's slash commands and keeps user edits.""" + expected = _repo_commands() + assert set(expected) == {"commune.md", "counsel.md", "inscribe.md", "recall.md"} + commands = tmp_path / ".opencode" / "commands" + commands.mkdir(parents=True) + (commands / "recall.md").write_text("my recall", encoding="utf-8") + + ok, msg = install_opencode( + str(tmp_path), bridge_config_root=tmp_path / "host-config" + ) + + assert ok, msg + assert "[exists] .opencode/commands/recall.md" in msg + assert "[create] .opencode/commands/commune.md" in msg + assert (commands / "recall.md").read_text(encoding="utf-8") == "my recall" + for name in ("commune.md", "counsel.md", "inscribe.md"): + assert (commands / name).read_text(encoding="utf-8") == expected[name] + + +def _repo_commands() -> dict[str, str]: + repo_commands = Path(__file__).resolve().parents[1] / ".opencode" / "commands" + return { + path.name: path.read_text(encoding="utf-8") + for path in repo_commands.glob("*.md") + } + + +def test_packaged_commands_match_the_repo_commands(): + """The shipped copies must not drift from the ones the repo uses.""" + assert _repo_commands() == COMMAND_TEMPLATES + + +def test_force_overwrites_commands_and_dry_run_writes_nothing(tmp_path): + ok, msg = install_opencode( + str(tmp_path), dry_run=True, bridge_config_root=tmp_path / "host-config" + ) + assert ok, msg + assert "[create] .opencode/commands/recall.md" in msg + assert not (tmp_path / ".opencode" / "commands" / "recall.md").exists() + + assert install_opencode(str(tmp_path), bridge_config_root=tmp_path / "host-config")[ + 0 + ] + recall = tmp_path / ".opencode" / "commands" / "recall.md" + recall.write_text("edited", encoding="utf-8") + + ok, msg = install_opencode( + str(tmp_path), force=True, bridge_config_root=tmp_path / "host-config" + ) + + assert ok, msg + assert "[overwrite] .opencode/commands/recall.md" in msg + assert recall.read_text(encoding="utf-8") == COMMAND_TEMPLATES["recall.md"] + + +def test_installer_refuses_to_write_through_a_symlink(tmp_path): + """A cloned repo must not steer the installer outside the project.""" + outside = tmp_path / "outside" + outside.mkdir() + project = tmp_path / "project" + (project / ".opencode" / "commands").mkdir(parents=True) + link = project / ".opencode" / "commands" / "recall.md" + try: + link.symlink_to(outside / "stolen.md") + except (OSError, NotImplementedError): + pytest.skip("symlinks are unavailable") + + ok, msg = install_opencode( + str(project), bridge_config_root=tmp_path / "host-config" + ) + + assert not ok + assert "link" in msg + assert not (outside / "stolen.md").exists() diff --git a/tests/test_retrieval_jobs.py b/tests/test_retrieval_jobs.py index 9bda81e..1b5e173 100644 --- a/tests/test_retrieval_jobs.py +++ b/tests/test_retrieval_jobs.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +import itertools import json import sqlite3 import tempfile +import threading import time import unittest from pathlib import Path @@ -391,13 +393,53 @@ def test_write_locked_builder_outlasts_heartbeat_busy_timeout(self): from daem0nmcp.retrieval.jobs import ProjectionJobRunner self._queue_job() + lease_duration_us = 1_000_000 + contended = threading.Event() + contentions = itertools.count(1) + + class _CountingConnection(sqlite3.Connection): + """Report when the heartbeat has really hit its busy timeout.""" + + def execute(self, sql, parameters=()): # type: ignore[override] + try: + return super().execute(sql, parameters) + except sqlite3.OperationalError: + if sql.startswith("BEGIN EXCLUSIVE") and next(contentions) >= 2: + contended.set() + raise + + def heartbeat_connection(): + return sqlite3.connect( + self.database_path, timeout=1.0, factory=_CountingConnection + ) def write_locked_build(_workspace_id): self.connection.execute("BEGIN IMMEDIATE") self.connection.execute( "UPDATE background_jobs SET result_json=result_json" ) - time.sleep(1.25) + claimed_lease_us = int( + self.connection.execute( + "SELECT lease_expires_at_us FROM background_jobs" + ).fetchone()[0] + ) + # Hold SQLite's writer lock until the heartbeat has provably been + # through its busy timeout more than once AND the lease it was + # claimed under has expired. Waiting on those two facts, rather + # than sleeping a duration chosen to outlast them, is what makes + # the contention this test is about deterministic under load. + deadline = time.monotonic() + 60 + while True: + expired = time.time_ns() // 1_000 > claimed_lease_us + if contended.is_set() and expired: + break + self.assertLess( + time.monotonic(), + deadline, + f"heartbeat never contended (contended={contended.is_set()}, " + f"lease expired={expired})", + ) + time.sleep(0.02) self.connection.commit() runner = ProjectionJobRunner( @@ -406,13 +448,16 @@ def write_locked_build(_workspace_id): clock_us=lambda: time.time_ns() // 1_000, lease_owner="test-worker", token_factory=lambda: "lease-token", - lease_duration_us=100_000, - heartbeat_interval_us=20_000, + lease_duration_us=lease_duration_us, + heartbeat_interval_us=50_000, + heartbeat_connection_factory=heartbeat_connection, retry_delay_us=1, ) result = runner.run_once() + self.assertTrue(contended.is_set()) + self.assertEqual("succeeded", result.status) self.assertEqual( ("succeeded", 1), diff --git a/tests/test_retrieval_projection_growth.py b/tests/test_retrieval_projection_growth.py new file mode 100644 index 0000000..64014fb --- /dev/null +++ b/tests/test_retrieval_projection_growth.py @@ -0,0 +1,570 @@ +"""Superseded projection generations are garbage-collected (KD-5, F-001). + +Every write rebuilds the local projections into a new generation. Without GC +each rebuild left a full-corpus FTS5 table, its document rows and a manifest +behind, so storage grew quadratically with the number of memories. +""" + +from __future__ import annotations + +import re +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +WORKSPACE_ID = "ws_0123456789abcdef01234567" +_NO_OPTIONAL_PROFILES = { + "local": "disabled", + "models-local": "disabled", + "graph": "disabled", +} +_LOCAL = ("lexical", "procedure", "outcome", "temporal") +# Spelled out here on purpose: importing the production mapping would make a +# shrunken GENERATION_TABLES pass this file. +_GENERATION_TABLES = { + "lexical": "retrieval_documents", + "outcome": "record_outcome_view", + "procedure": "record_procedures", +} + + +def _record(content: str, *, procedure: bool = False) -> dict[str, object]: + return { + "record_type": "procedure" if procedure else "decision", + "legacy_type": None, + "content": content, + "rationale": None, + "context": {"steps": ["Check state.", "Apply change."]} if procedure else {}, + "tags": ["growth"], + "file_path": None, + "file_path_relative": None, + "keywords": None, + "is_permanent": False, + "pinned": False, + "archived": False, + "outcome": None, + "worked": None, + "recall_count": 0, + "surprise_score": None, + "importance_score": None, + "source_client": "growth-test", + "source_model": None, + "deleted_at_us": None, + } + + +class ProjectionGenerationGrowthTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + from daem0nmcp.migrations.schema import MIGRATIONS + + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "growth.sqlite3" + self.connection = sqlite3.connect(self.path) + self.connection.execute("PRAGMA foreign_keys=ON") + # Pre-PR-8 databases on POSIX still use the rollback journal, where + # readers block a committing writer; GC must stay safe there. + self.connection.execute("PRAGMA journal_mode=DELETE") + # Durability is irrelevant here; skipping fsync keeps the hundreds of + # seeding rebuilds fast on slow CI disks. + self.connection.execute("PRAGMA synchronous=OFF") + for version, _description, statements in MIGRATIONS: + if 16 <= version <= 18: + for statement in statements: + self.connection.execute(statement) + self.connection.commit() + self._sequence = 0 + + def tearDown(self) -> None: + self.connection.close() + self._directory.cleanup() + + def _record_outcome(self, record_id: str) -> None: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, + stream_kind="memory", + event_type="memory.outcome_recorded", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": { + **_record("growth memory number 1", procedure=True), + "outcome": "the generation was collected", + "worked": True, + } + }, + ) + ) + self.connection.commit() + + def _append(self, *, procedure: bool = False) -> str: + from daem0nmcp.event_store import EventCommand, EventStore + + self._sequence += 1 + record_id = f"mem_{self._sequence:064x}" + EventStore(self.connection).append_and_project( + EventCommand( + workspace_id=WORKSPACE_ID, + stream_id=record_id, + stream_kind="memory", + event_type="memory.created", + occurred_at_us=100 + self._sequence, + recorded_at_us=100 + self._sequence, + actor_type="system", + payload={ + "record": _record( + f"growth memory number {self._sequence}", procedure=procedure + ) + }, + ) + ) + self.connection.commit() + return record_id + + def _rebuild(self, projection: str) -> None: + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + from daem0nmcp.retrieval.specialized_projection import ( + SpecializedProjectionBuilder, + ) + + if projection == "lexical": + LexicalProjectionBuilder(self.connection).rebuild(WORKSPACE_ID) + else: + SpecializedProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID, projection, force=True + ) + + def _manifests(self, projection: str) -> dict[int, str]: + return { + int(row[0]): str(row[1]) + for row in self.connection.execute( + "SELECT generation,status FROM projection_manifests " + "WHERE workspace_id=? AND projection_name=?", + (WORKSPACE_ID, projection), + ) + } + + def _fts_generations(self, projection: str) -> set[int]: + prefix = ( + "retrieval_procedure_fts_" + if projection == "procedure" + else ("retrieval_fts_") + ) + pattern = re.compile(re.escape(prefix + WORKSPACE_ID[3:]) + r"_g(\d+)") + return { + int(match[1]) + for (name,) in self.connection.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + if (match := pattern.fullmatch(str(name))) is not None + } + + def _active(self, projection: str) -> int: + [generation] = [ + generation + for generation, status in self._manifests(projection).items() + if status == "active" + ] + return generation + + async def test_spaced_writes_keep_generations_bounded(self) -> None: + from daem0nmcp.retrieval.runtime import drain_projection_jobs + + first = self._append(procedure=True) + self._record_outcome(first) + for projection in _LOCAL: + self._rebuild(projection) + writes = 20 + for index in range(writes): + self._append(procedure=index % 5 == 0) + # Every job is due immediately: this is the interactive case of + # writes spaced further apart than the coalescing grace. + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runs = await drain_projection_jobs( + self.path, + config=SimpleNamespace(), + max_jobs=8, + # The real drain: every local projection rebuilds, so the + # specialized GC call site runs on the production path. + include_optional=True, + capability_statuses=_NO_OPTIONAL_PROFILES, + ) + self.assertEqual( + {"succeeded"}, + {run.status for run in runs}, + [run.reason for run in runs], + ) + self.assertEqual( + set(_LOCAL), + {name for run in runs for name in run.projections}, + ) + + for projection in _LOCAL: + manifests = self._manifests(projection) + self.assertLessEqual(len(manifests), 2, (projection, manifests)) + self.assertEqual(1, list(manifests.values()).count("active"), projection) + for projection in ("lexical", "procedure"): + tables = self._fts_generations(projection) + self.assertLessEqual(len(tables), 2, (projection, tables)) + self.assertEqual(tables, set(self._manifests(projection)), projection) + live = self.connection.execute( + "SELECT COUNT(*) FROM memory_records WHERE workspace_id=?", + (WORKSPACE_ID,), + ).fetchone()[0] + self.assertEqual(writes + 1, live) + documents = self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents" + ).fetchone()[0] + self.assertLessEqual(documents, 2 * live) + self._assert_generation_rows_collected() + # The stale marker every write sets clears once the rebuild runs, and + # GC never leaves it stuck on an active manifest. + self.assertEqual( + [], + self.connection.execute( + "SELECT projection_name FROM projection_manifests " + "WHERE workspace_id=? AND status='active' AND (" + "json_extract(details_json,'$.rebuild_required_event_id') IS NOT NULL " + "OR json_extract(details_json,'$.rebuild_required_at_us') IS NOT NULL)", + (WORKSPACE_ID,), + ).fetchall(), + ) + + def _assert_generation_rows_collected(self) -> None: + """Every generation table keeps rows only for surviving manifests.""" + + for projection, table in _GENERATION_TABLES.items(): + rows = { + int(row[0]) + for row in self.connection.execute( + f'SELECT DISTINCT projection_generation FROM "{table}" ' + "WHERE workspace_id=?", + (WORKSPACE_ID,), + ) + } + self.assertTrue(rows, f"{table} has no rows to prove collection") + self.assertLessEqual(rows, set(self._manifests(projection)), table) + + def test_stale_backlog_shrinks_five_generations_per_activation(self) -> None: + self._append(procedure=True) + for projection in ("lexical", "procedure"): + with self.subTest(projection=projection): + # Seed 120 superseded generations the way a pre-GC database + # accumulated them. + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(121): + self._rebuild(projection) + self.assertEqual(121, len(self._manifests(projection))) + self.assertEqual(121, len(self._fts_generations(projection))) + + previous = 120 + while previous > 1: + self._rebuild(projection) + stale = sum( + status != "active" + for status in self._manifests(projection).values() + ) + self.assertEqual(max(1, previous + 1 - 5), stale) + previous = stale + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests)) + self.assertEqual(set(manifests), self._fts_generations(projection)) + active = self._active(projection) + self.assertEqual({active - 1, active}, set(manifests)) + + def test_failed_or_unactivated_generation_never_evicts_last_active(self) -> None: + from daem0nmcp.event_store import deterministic_id + + self._append() + self._rebuild("lexical") + self._rebuild("lexical") + previous_active = self._active("lexical") + template = self.connection.execute( + "SELECT source_event_count,source_event_root_hash,row_count," + "builder_version FROM projection_manifests WHERE workspace_id=? " + "AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, previous_active), + ).fetchone() + # Generations just below the next one that never became active: an + # arithmetic "keep the one below the new generation" rule would keep + # these and evict the generation readers were actually using. + for offset, status in ((1, "failed"), (2, "ready")): + generation = previous_active + offset + self.connection.execute( + "INSERT INTO projection_manifests (manifest_id,workspace_id," + "projection_name,generation,projection_version,status," + "source_event_count,source_event_root_hash,row_count," + "builder_version,details_json,started_at_us) " + "VALUES (?,?,'lexical',?,1,?,?,?,?,?,'{}',1)", + ( + deterministic_id("prj", "growth-test", generation), + WORKSPACE_ID, + generation, + status, + *template, + ), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + active = self._active("lexical") + self.assertEqual(previous_active + 3, active) + self.assertEqual({previous_active: "ready", active: "active"}, manifests) + self.assertEqual({previous_active, active}, self._fts_generations("lexical")) + + def test_gc_blocked_by_an_open_reader_does_not_fail_the_rebuild(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + reader = sqlite3.connect(self.path, check_same_thread=False) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + # A reader that looked up an old generation is mid-query when + # the activation's GC starts. + reader.execute("BEGIN") + reader.execute( + f'SELECT COUNT(*) FROM "{lexical_fts_table_name(WORKSPACE_ID, oldest)}"' + ).fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + self.assertEqual([0], collected) + self.assertIn(oldest, self._manifests("lexical")) + self.assertIn(oldest, self._fts_generations("lexical")) + self.assertFalse(self.connection.in_transaction) + finally: + reader.close() + + # The next activation retries and catches up. + self._rebuild("lexical") + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertLessEqual(len(self._manifests("lexical")), 2) + + def test_gc_in_wal_mode_collects_under_an_open_reader(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self.connection.execute("PRAGMA journal_mode=WAL") + self._append() + for _ in range(3): + self._rebuild("lexical") + oldest = min(self._manifests("lexical")) + table = lexical_fts_table_name(WORKSPACE_ID, oldest) + reader = sqlite3.connect(self.path) + collect = projections.collect_superseded_generations + collected: list[int] = [] + + def collect_while_reading(*args, **kwargs): + reader.execute("BEGIN") + reader.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() + collected.append(collect(*args, **kwargs)) + return collected[-1] + + try: + with patch.object( + projections, + "collect_superseded_generations", + collect_while_reading, + ): + result = projections.LexicalProjectionBuilder(self.connection).rebuild( + WORKSPACE_ID + ) + self.assertEqual("active", result.status) + # WAL readers do not block the drop, and the reader's snapshot + # still answers from the generation being collected. + self.assertEqual([1], collected) + self.assertEqual( + 1, len(reader.execute(f'SELECT * FROM "{table}" LIMIT 1').fetchall()) + ) + self.assertNotIn(oldest, self._manifests("lexical")) + self.assertNotIn(oldest, self._fts_generations("lexical")) + finally: + reader.close() + + def test_repair_left_rebuild_required_generations_are_collected(self) -> None: + self._append() + self._rebuild("lexical") + repaired = self._active("lexical") + # verify-v7 --repair-projections demotes the active generation and + # rebuilds in one transaction, so its GC is a no-op. + self.connection.execute( + "UPDATE projection_manifests SET status='rebuild_required' " + "WHERE workspace_id=? AND projection_name='lexical' AND generation=?", + (WORKSPACE_ID, repaired), + ) + self.connection.commit() + + self._rebuild("lexical") + + manifests = self._manifests("lexical") + self.assertNotIn(repaired, manifests) + self.assertEqual(set(manifests), self._fts_generations("lexical")) + self.assertEqual( + 0, + self.connection.execute( + "SELECT COUNT(*) FROM retrieval_documents WHERE workspace_id=? " + "AND projection_generation=?", + (WORKSPACE_ID, repaired), + ).fetchone()[0], + ) + + def test_an_undroppable_generation_does_not_wedge_the_rest(self) -> None: + from daem0nmcp.retrieval import projections + from daem0nmcp.retrieval.lexical_config import lexical_fts_table_name + + self._append() + with patch.object( + projections, "collect_superseded_generations", return_value=0 + ): + for _ in range(4): + self._rebuild("lexical") + generations = sorted(self._manifests("lexical")) + damaged = generations[0] + original = projections._drop_generation + + def refuse_oldest( + connection, workspace_id, projection, manifest_id, generation + ): + if generation == damaged: + raise sqlite3.OperationalError("malformed database schema") + return original( + connection, workspace_id, projection, manifest_id, generation + ) + + with patch.object(projections, "_drop_generation", refuse_oldest): + collected = projections.collect_superseded_generations( + self.connection, WORKSPACE_ID, "lexical" + ) + + manifests = self._manifests("lexical") + # The damaged generation is skipped, not retried forever, and every + # later victim is still collected. + self.assertGreaterEqual(collected, 1) + self.assertIn(damaged, manifests) + self.assertIn(damaged, self._fts_generations("lexical")) + self.assertNotIn(generations[1], manifests) + self.assertNotIn(generations[1], self._fts_generations("lexical")) + self.assertIsNotNone(lexical_fts_table_name(WORKSPACE_ID, damaged)) + + def test_offline_compaction_drains_a_backlog_and_reclaims_the_file(self) -> None: + from daem0nmcp.retrieval.operations import compact_projections + + self._append(procedure=True) + with ( + patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ), + patch( + "daem0nmcp.retrieval.specialized_projection." + "collect_superseded_generations", + return_value=0, + ), + ): + for _ in range(30): + for projection in ("lexical", "procedure"): + self._rebuild(projection) + self.assertEqual(30, len(self._manifests("lexical"))) + before = self.path.stat().st_size + + payload = compact_projections(self.connection, WORKSPACE_ID) + + self.assertEqual(28, payload["collected"]["lexical"]) + self.assertEqual(28, payload["collected"]["procedure"]) + self.assertTrue(payload["vacuumed"]) + self.assertEqual(58, payload["superseded_manifests_before"]) + self.assertEqual(2, payload["superseded_manifests_after"]) + for projection in ("lexical", "procedure"): + manifests = self._manifests(projection) + self.assertEqual(2, len(manifests), projection) + self.assertEqual(set(manifests), self._fts_generations(projection)) + self.assertLess(self.path.stat().st_size, before) + + def test_lock_contention_is_retried_not_dead_lettered(self) -> None: + from daem0nmcp.retrieval.jobs import ProjectionJobRunner + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + self._append() + self.connection.execute("UPDATE background_jobs SET available_at_us=0") + self.connection.commit() + runner_connection = sqlite3.connect(self.path, timeout=0.1) + blocker = sqlite3.connect(self.path, check_same_thread=False) + lexical = LexicalProjectionBuilder(runner_connection) + + def contended_rebuild(workspace_id: str): + blocker.execute("BEGIN IMMEDIATE") + try: + return lexical.rebuild(workspace_id) + finally: + blocker.rollback() + + try: + run = ProjectionJobRunner( + runner_connection, builders={"lexical": contended_rebuild} + ).run_once() + assert run is not None + self.assertEqual("DATABASE_IN_USE", run.reason) + self.assertEqual("queued", run.status) + finally: + blocker.close() + runner_connection.close() + + +class GenerationGcUnitTests(unittest.TestCase): + def test_gc_inside_a_caller_transaction_is_a_no_op(self) -> None: + from daem0nmcp.retrieval.projections import collect_superseded_generations + + connection = sqlite3.connect(":memory:") + try: + connection.execute("CREATE TABLE t(x)") + connection.execute("INSERT INTO t VALUES (1)") + self.assertTrue(connection.in_transaction) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "lexical") + ) + self.assertEqual( + 0, collect_superseded_generations(connection, WORKSPACE_ID, "graph") + ) + finally: + connection.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_start_server.py b/tests/test_start_server.py new file mode 100644 index 0000000..f53604b --- /dev/null +++ b/tests/test_start_server.py @@ -0,0 +1,142 @@ +"""The HTTP launchers serve the user's project, never the Daem0n-MCP checkout.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest + +# Import before patch.dict(sys.modules) snapshots it, so the patched launcher is +# the one start_server imports in every test (not a fresh re-import). +import daem0nmcp.api.v7.launcher # noqa: F401 + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _launched_project_root(argv: list[str], environ: dict[str, str]) -> str: + import start_server + + seen: list[str] = [] + + def create_server(transport: str, *, host: str | None = None) -> object: + seen.append(os.environ["DAEM0NMCP_PROJECT_ROOT"]) + return object() + + fake_server = types.ModuleType("daem0nmcp.server") + fake_server.create_server = create_server + with ( + patch.dict(sys.modules, {"daem0nmcp.server": fake_server}), + patch.object(sys, "argv", ["start_server.py", *argv]), + patch.dict(os.environ, environ, clear=True), + patch("daem0nmcp.api.v7.launcher.run_server"), + ): + start_server.main() + return seen[0] + + +def test_exported_project_root_survives(tmp_path): + exported = tmp_path / "exported" + exported.mkdir() + root = _launched_project_root([], {"DAEM0NMCP_PROJECT_ROOT": str(exported)}) + assert root == str(exported.resolve()) + + +def test_project_argument_wins_over_exported_root(tmp_path): + chosen = tmp_path / "chosen" + chosen.mkdir() + (tmp_path / "exported").mkdir() + root = _launched_project_root( + ["--project", str(chosen)], + {"DAEM0NMCP_PROJECT_ROOT": str(tmp_path / "exported")}, + ) + assert root == str(chosen.resolve()) + + +def test_current_directory_is_the_default(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + assert _launched_project_root([], {}) == str(tmp_path.resolve()) + + +def test_a_missing_project_directory_is_refused(tmp_path): + """A typo must not create and serve an empty workspace.""" + with pytest.raises(SystemExit) as raised: + _launched_project_root(["--project", str(tmp_path / "typo")], {}) + assert raised.value.code == 2 + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +@pytest.mark.parametrize("use_argument", [False, True]) +def test_batch_launcher_passes_the_callers_project(tmp_path, use_argument): + """The .bat changes to the repo dir, so it must capture the project first.""" + recorded, environment = _shim(tmp_path) + project = tmp_path / "my project (x86) ^ !name!" + project.mkdir() + + _run_batch( + ["my project (x86) ^ !name!"] if use_argument else [], + cwd=tmp_path if use_argument else project, + environment=environment, + ) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == f'start_server.py --port 9876 --project "{project}\\."' + assert Path(f"{project}\\.").resolve() == project.resolve() + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +def test_batch_launcher_leaves_an_exported_project_root_alone(tmp_path): + recorded, environment = _shim(tmp_path) + exported = tmp_path / "exported" + exported.mkdir() + environment["DAEM0NMCP_PROJECT_ROOT"] = str(exported) + + _run_batch([], cwd=tmp_path, environment=environment) + + arguments = recorded.read_text(encoding="ascii").strip() + assert arguments == "start_server.py --port 9876" + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows batch launcher") +def test_batch_launcher_refuses_the_windows_system_directory(tmp_path): + """ "Run as administrator" starts in System32; that is never the project.""" + recorded, environment = _shim(tmp_path) + system32 = Path(os.environ["SYSTEMROOT"]) / "System32" + + completed = _run_batch([], cwd=system32, environment=environment, check=False) + + assert completed.returncode == 2 + assert not recorded.exists() + + +def _shim(tmp_path: Path) -> tuple[Path, dict[str, str]]: + """Put a python.cmd on PATH that records the arguments it is given.""" + shim = tmp_path / "shim" + shim.mkdir() + recorded = tmp_path / "argv.txt" + (shim / "python.cmd").write_text(f'@echo %*> "{recorded}"\r\n', encoding="ascii") + environment = {**os.environ, "PATH": f"{shim}{os.pathsep}{os.environ['PATH']}"} + environment.pop("DAEM0NMCP_PROJECT_ROOT", None) + return recorded, environment + + +def _run_batch( + arguments: list[str], + *, + cwd: Path, + environment: dict[str, str], + check: bool = True, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + ["cmd", "/c", str(REPO_ROOT / "start_daem0nmcp_server.bat"), *arguments], + cwd=cwd, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + check=check, + timeout=30, + ) diff --git a/tests/test_vectors.py b/tests/test_vectors.py index ac78d0e..f5420a9 100644 --- a/tests/test_vectors.py +++ b/tests/test_vectors.py @@ -62,16 +62,16 @@ def test_hybrid_fallback_to_tfidf(self): class TestEncodeDecode: """Test vector encoding and decoding.""" - def test_encode_reports_models_local_remediation_when_unavailable(self): + def test_encode_reports_models_local_remediation_when_unavailable( + self, monkeypatch + ): """The disabled models-local profile exposes its explicit remediation.""" - if not vectors.is_available(): - from daem0nmcp.capabilities import CapabilityUnavailableError - - with pytest.raises( - CapabilityUnavailableError, match="models-local" - ) as error: - vectors.encode("test text") - assert "DAEM0NMCP_MODELS_LOCAL_ENABLED" in str(error.value) + from daem0nmcp.capabilities import CapabilityUnavailableError + + monkeypatch.setenv("DAEM0NMCP_MODELS_LOCAL_ENABLED", "false") + with pytest.raises(CapabilityUnavailableError, match="models-local") as error: + vectors.encode("test text") + assert "DAEM0NMCP_MODELS_LOCAL_ENABLED" in str(error.value) def test_decode_empty_returns_none(self): """decode returns None for empty bytes.""" diff --git a/tests/test_verify_v7.py b/tests/test_verify_v7.py index c50ed20..3ad9705 100644 --- a/tests/test_verify_v7.py +++ b/tests/test_verify_v7.py @@ -7,6 +7,7 @@ import subprocess import sys from pathlib import Path +from unittest.mock import patch import pytest @@ -198,6 +199,32 @@ def test_verify_v7_replays_every_authority_domain(tmp_path: Path) -> None: assert report["checks"]["active_pointer"]["generation"] == 1 +def test_verify_v7_reports_superseded_generation_counts(tmp_path: Path) -> None: + """A GC that has stopped making progress must be visible without logs.""" + from daem0nmcp.retrieval.projections import LexicalProjectionBuilder + + storage, database = _storage(tmp_path) + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys=ON") + with patch( + "daem0nmcp.retrieval.projections.collect_superseded_generations", + return_value=0, + ): + for _ in range(3): + LexicalProjectionBuilder(connection).rebuild(WORKSPACE_ID) + connection.commit() + + report = verify_v7(storage, WORKSPACE_ID) + + inventory = report["checks"]["projection_generations"] + assert inventory["ok"] is True + # Three generations: one active, two superseded and uncollected. + assert inventory["lexical"] == 2 + assert inventory["outcome"] == 0 + # Only the partitions themselves, never their FTS5 shadow tables. + assert inventory["fts_partitions"] == 3 + + def test_verify_v7_accepts_intact_active_code_generation(tmp_path: Path) -> None: storage, database = _storage(tmp_path) _build_code_projection(database)